diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index 033ab804b165eada6c0de4972b102360d75d8525..fd603de611e56203ab7fcdb32c9ff6ad7111a2a9 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -153,7 +153,6 @@ function exclude-linux() {
   for project in ${projects}; do
     case ${project} in
     cross-project-tests) ;; # tests failing
-    lldb)                ;; # tests failing
     openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
     *)
       echo "${project}"
@@ -170,7 +169,7 @@ function exclude-windows() {
     compiler-rt)         ;; # tests taking too long
     openmp)              ;; # TODO: having trouble with the Perl installation
     libc)                ;; # no Windows support
-    lldb)                ;; # tests failing
+    lldb)                ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
     bolt)                ;; # tests are not supported yet
     *)
       echo "${project}"
@@ -213,7 +212,7 @@ function check-targets() {
       echo "check-unwind"
     ;;
     lldb)
-      echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
+      echo "check-lldb"
     ;;
     pstl)
       echo "check-all"
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 38d7128f241b6756f6b20652261a6f055a244915..b78dc59432b65c9dcdf5130fe6aea85fb2d03148 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -39,6 +39,7 @@ targets="${2}"
 
 echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
+pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
diff --git a/.github/workflows/ci-post-commit-analyzer-run.py b/.github/workflows/ci-post-commit-analyzer-run.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5f52d3b2fa6719ac741866dd1a7f678189e94a5
--- /dev/null
+++ b/.github/workflows/ci-post-commit-analyzer-run.py
@@ -0,0 +1,34 @@
+import json
+import multiprocessing
+import os
+import re
+import subprocess
+import sys
+
+
+def run_analyzer(data):
+    os.chdir(data["directory"])
+    command = (
+        data["command"]
+        + f" --analyze --analyzer-output html -o analyzer-results -Xclang -analyzer-config -Xclang max-nodes=75000"
+    )
+    print(command)
+    subprocess.run(command, shell=True, check=True)
+
+
+def pool_error(e):
+    print("Error analyzing file:", e)
+
+
+def main():
+    db_path = sys.argv[1]
+    database = json.load(open(db_path))
+
+    with multiprocessing.Pool() as pool:
+        pool.map_async(run_analyzer, [k for k in database], error_callback=pool_error)
+        pool.close()
+        pool.join()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/ci-post-commit-analyzer.yml b/.github/workflows/ci-post-commit-analyzer.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d614dd07b3a493efe6412165304a3c6f2a22940b
--- /dev/null
+++ b/.github/workflows/ci-post-commit-analyzer.yml
@@ -0,0 +1,95 @@
+name: Post-Commit Static Analyzer
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - 'release/**'
+    paths:
+      - 'clang/**'
+      - 'llvm/**'
+      - '.github/workflows/ci-post-commit-analyzer.yml'
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - closed
+    paths:
+      - '.github/workflows/ci-post-commit-analyzer.yml'
+      - '.github/workflows/ci-post-commit-analyzer-run.py'
+  schedule:
+    - cron: '30 0 * * *'
+
+concurrency:
+  group: >-
+    llvm-project-${{ github.workflow }}-${{ github.event_name == 'pull_request' &&
+      ( github.event.pull_request.number || github.ref) }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  post-commit-analyzer:
+    if: >-
+      github.repository_owner == 'llvm' &&
+      github.event.action != 'closed'
+    runs-on: ubuntu-22.04
+    container:
+      image: 'ghcr.io/llvm/ci-ubuntu-22.04:latest'
+    env:
+      LLVM_VERSION: 18
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          # A full build of llvm, clang, lld, and lldb takes about 250MB
+          # of ccache space. There's not much reason to have more than this,
+          # because we usually won't need to save cache entries from older
+          # builds.  Also, there is an overall 10GB cache limit, and each
+          # run creates a new cache entry so we want to ensure that we have
+          # enough cache space for all the tests to run at once and still
+          # fit under the 10 GB limit.
+          # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
+          max-size: 2G
+          key: post-commit-analyzer
+          variant: sccache
+
+      - name: Configure
+        run: |
+              cmake -B build -S llvm -G Ninja \
+                  -DLLVM_ENABLE_ASSERTIONS=ON \
+                  -DLLVM_ENABLE_PROJECTS=clang \
+                  -DLLVM_BUILD_LLVM_DYLIB=ON \
+                  -DLLVM_LINK_LLVM_DYLIB=ON \
+                  -DCMAKE_CXX_COMPILER=clang++ \
+                  -DCMAKE_C_COMPILER=clang \
+                  -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+                  -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+                  -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+                  -DLLVM_INCLUDE_TESTS=OFF \
+                  -DCLANG_INCLUDE_TESTS=OFF \
+                  -DCMAKE_BUILD_TYPE=Release
+
+      - name: Build
+        run: |
+          # FIXME: We need to build all the generated header files in order to be able to run
+          # the analyzer on every file.  Building libLLVM and libclang is probably overkill for
+          # this, but it's better than building every target.
+          ninja -v -C build libLLVM.so libclang.so
+
+          # Run the analyzer.
+          python3 .github/workflows/ci-post-commit-analyzer-run.py build/compile_commands.json
+
+          scan-build --generate-index-only build/analyzer-results
+
+      - name: Upload Results
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+        if: always()
+        with:
+          name: analyzer-results
+          path: 'build/analyzer-results/*'
+
diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
index fbc4548e6636e8f8c7313c27bfea12ae03556736..8c6bcf46384105a291250efa99b9a5362dcd52f8 100644
--- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile
+++ b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
@@ -37,7 +37,7 @@ RUN cmake -B ./build -G Ninja ./llvm \
   -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
   -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
   -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
-  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
   -DCLANG_DEFAULT_LINKER="lld" \
   -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
 
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 8fa3bf3d9df51a8fd4e4df457547bb4d292a58a7..7de4d00334d14a5b9914b6c2f54b41b2be58ac07 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -216,7 +216,7 @@ jobs:
 
     - id: package-info
       run: |
-        filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.gz"
+        filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.xz"
         echo "filename=$filename" >> $GITHUB_OUTPUT
         echo "path=/mnt/build/tools/clang/stage2-bins/$filename" >> $GITHUB_OUTPUT
 
diff --git a/bolt/include/bolt/Core/GDBIndex.h b/bolt/include/bolt/Core/GDBIndex.h
new file mode 100644
index 0000000000000000000000000000000000000000..6604c2a11472d47c8070b3cc4a1c5a26c71b47e0
--- /dev/null
+++ b/bolt/include/bolt/Core/GDBIndex.h
@@ -0,0 +1,61 @@
+//===-- bolt/Core/GDBIndex.h - GDB Index support ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file contains declaration of classes required for generation of
+/// .gdb_index section.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_CORE_GDB_INDEX_H
+#define BOLT_CORE_GDB_INDEX_H
+
+#include "bolt/Core/BinaryContext.h"
+#include <vector>
+
+namespace llvm {
+namespace bolt {
+
+class GDBIndex {
+public:
+  /// Contains information about TU so we can write out correct entries in GDB
+  /// index.
+  struct GDBIndexTUEntry {
+    uint64_t UnitOffset;
+    uint64_t TypeHash;
+    uint64_t TypeDIERelativeOffset;
+  };
+
+private:
+  BinaryContext &BC;
+
+  /// Entries for GDB Index Types CU List.
+  using GDBIndexTUEntryType = std::vector<GDBIndexTUEntry>;
+  GDBIndexTUEntryType GDBIndexTUEntryVector;
+
+public:
+  GDBIndex(BinaryContext &BC) : BC(BC) {}
+
+  std::mutex GDBIndexMutex;
+
+  /// Adds an GDBIndexTUEntry if .gdb_index section exists.
+  void addGDBTypeUnitEntry(const GDBIndexTUEntry &&Entry);
+
+  /// Rewrite .gdb_index section if present.
+  void updateGdbIndexSection(const CUOffsetMap &CUMap, const uint32_t NumCUs,
+                             DebugARangesSectionWriter &ARangesSectionWriter);
+
+  /// Returns all entries needed for Types CU list.
+  const GDBIndexTUEntryType &getGDBIndexTUEntryVector() const {
+    return GDBIndexTUEntryVector;
+  }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index f7cf538bd0e867eba4f11636b45361a2280c4706..a5fb3901428d9d393cca8dd881318e6c5401dbf9 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1706,12 +1706,9 @@ public:
   }
 
   /// Reverses the branch condition in Inst and update its taken target to TBB.
-  ///
-  /// Returns true on success.
-  virtual bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
+  virtual void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
                                       MCContext *Ctx) const {
     llvm_unreachable("not implemented");
-    return false;
   }
 
   virtual bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB,
@@ -1751,12 +1748,9 @@ public:
   }
 
   /// Sets the taken target of the branch instruction to Target.
-  ///
-  /// Returns true on success.
-  virtual bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
+  virtual void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
                                    MCContext *Ctx) const {
     llvm_unreachable("not implemented");
-    return false;
   }
 
   /// Extract a symbol and an addend out of the fixup value expression.
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 0b44acb0816f2fab0c627ae7e451a7782732de53..5793963f9b80de23712e227a30a3c34acabb18b2 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -194,6 +194,7 @@ private:
 
 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
   Streamer.initSections(false, *BC.STI);
+  Streamer.setUseAssemblerInfoForParsing(false);
 
   if (opts::UpdateDebugSections && BC.isELF()) {
     // Force the emission of debug line info into allocatable section to ensure
@@ -226,6 +227,7 @@ void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
   // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.
   if (BC.isELF())
     AddressMap::emit(Streamer, BC);
+  Streamer.setUseAssemblerInfoForParsing(true);
 }
 
 void BinaryEmitter::emitFunctions() {
diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt
index 441df9fe0846481c8e88eb74f9d1cbff96b4d594..873cf67a56291f7a1f760f83e4f81c5dcb3fe056 100644
--- a/bolt/lib/Core/CMakeLists.txt
+++ b/bolt/lib/Core/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_library(LLVMBOLTCore
   DynoStats.cpp
   Exceptions.cpp
   FunctionLayout.cpp
+  GDBIndex.cpp
   HashUtilities.cpp
   JumpTable.cpp
   MCPlusBuilder.cpp
diff --git a/bolt/lib/Core/GDBIndex.cpp b/bolt/lib/Core/GDBIndex.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9e6d24167d559e96f81c646775082523c88b4a73
--- /dev/null
+++ b/bolt/lib/Core/GDBIndex.cpp
@@ -0,0 +1,185 @@
+//===- bolt/Core/GDBIndex.cpp  - GDB Index support ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Core/GDBIndex.h"
+
+using namespace llvm::bolt;
+using namespace llvm::support::endian;
+
+void GDBIndex::addGDBTypeUnitEntry(const GDBIndexTUEntry &&Entry) {
+  std::lock_guard<std::mutex> Lock(GDBIndexMutex);
+  if (!BC.getGdbIndexSection())
+    return;
+  GDBIndexTUEntryVector.emplace_back(Entry);
+}
+
+void GDBIndex::updateGdbIndexSection(
+    const CUOffsetMap &CUMap, const uint32_t NumCUs,
+    DebugARangesSectionWriter &ARangesSectionWriter) {
+  if (!BC.getGdbIndexSection())
+    return;
+
+  // See https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html
+  // for .gdb_index section format.
+
+  StringRef GdbIndexContents = BC.getGdbIndexSection()->getContents();
+
+  const char *Data = GdbIndexContents.data();
+
+  // Parse the header.
+  const uint32_t Version = read32le(Data);
+  if (Version != 7 && Version != 8) {
+    errs() << "BOLT-ERROR: can only process .gdb_index versions 7 and 8\n";
+    exit(1);
+  }
+
+  // Some .gdb_index generators use file offsets while others use section
+  // offsets. Hence we can only rely on offsets relative to each other,
+  // and ignore their absolute values.
+  const uint32_t CUListOffset = read32le(Data + 4);
+  const uint32_t CUTypesOffset = read32le(Data + 8);
+  const uint32_t AddressTableOffset = read32le(Data + 12);
+  const uint32_t SymbolTableOffset = read32le(Data + 16);
+  const uint32_t ConstantPoolOffset = read32le(Data + 20);
+  Data += 24;
+
+  // Map CUs offsets to indices and verify existing index table.
+  std::map<uint32_t, uint32_t> OffsetToIndexMap;
+  const uint32_t CUListSize = CUTypesOffset - CUListOffset;
+  const uint32_t TUListSize = AddressTableOffset - CUTypesOffset;
+  const unsigned NUmCUsEncoded = CUListSize / 16;
+  unsigned MaxDWARFVersion = BC.DwCtx->getMaxVersion();
+  unsigned NumDWARF5TUs =
+      getGDBIndexTUEntryVector().size() - BC.DwCtx->getNumTypeUnits();
+  bool SkipTypeUnits = false;
+  // For DWARF5 Types are in .debug_info.
+  // LLD doesn't generate Types CU List, and in CU list offset
+  // only includes CUs.
+  // GDB 11+ includes only CUs in CU list and generates Types
+  // list.
+  // GDB 9 includes CUs and TUs in CU list and generates TYpes
+  // list. The NumCUs is CUs + TUs, so need to modify the check.
+  // For split-dwarf
+  // GDB-11, DWARF5: TU units from dwo are not included.
+  // GDB-11, DWARF4: TU units from dwo are included.
+  if (MaxDWARFVersion >= 5)
+    SkipTypeUnits = !TUListSize ? true
+                                : ((NUmCUsEncoded + NumDWARF5TUs) ==
+                                   BC.DwCtx->getNumCompileUnits());
+
+  if (!((CUListSize == NumCUs * 16) ||
+        (CUListSize == (NumCUs + NumDWARF5TUs) * 16))) {
+    errs() << "BOLT-ERROR: .gdb_index: CU count mismatch\n";
+    exit(1);
+  }
+  DenseSet<uint64_t> OriginalOffsets;
+  for (unsigned Index = 0, Units = BC.DwCtx->getNumCompileUnits();
+       Index < Units; ++Index) {
+    const DWARFUnit *CU = BC.DwCtx->getUnitAtIndex(Index);
+    if (SkipTypeUnits && CU->isTypeUnit())
+      continue;
+    const uint64_t Offset = read64le(Data);
+    Data += 16;
+    if (CU->getOffset() != Offset) {
+      errs() << "BOLT-ERROR: .gdb_index CU offset mismatch\n";
+      exit(1);
+    }
+
+    OriginalOffsets.insert(Offset);
+    OffsetToIndexMap[Offset] = Index;
+  }
+
+  // Ignore old address table.
+  const uint32_t OldAddressTableSize = SymbolTableOffset - AddressTableOffset;
+  // Move Data to the beginning of symbol table.
+  Data += SymbolTableOffset - CUTypesOffset;
+
+  // Calculate the size of the new address table.
+  uint32_t NewAddressTableSize = 0;
+  for (const auto &CURangesPair : ARangesSectionWriter.getCUAddressRanges()) {
+    const SmallVector<DebugAddressRange, 2> &Ranges = CURangesPair.second;
+    NewAddressTableSize += Ranges.size() * 20;
+  }
+
+  // Difference between old and new table (and section) sizes.
+  // Could be negative.
+  int32_t Delta = NewAddressTableSize - OldAddressTableSize;
+
+  size_t NewGdbIndexSize = GdbIndexContents.size() + Delta;
+
+  // Free'd by ExecutableFileMemoryManager.
+  auto *NewGdbIndexContents = new uint8_t[NewGdbIndexSize];
+  uint8_t *Buffer = NewGdbIndexContents;
+
+  write32le(Buffer, Version);
+  write32le(Buffer + 4, CUListOffset);
+  write32le(Buffer + 8, CUTypesOffset);
+  write32le(Buffer + 12, AddressTableOffset);
+  write32le(Buffer + 16, SymbolTableOffset + Delta);
+  write32le(Buffer + 20, ConstantPoolOffset + Delta);
+  Buffer += 24;
+
+  using MapEntry = std::pair<uint32_t, CUInfo>;
+  std::vector<MapEntry> CUVector(CUMap.begin(), CUMap.end());
+  // Need to sort since we write out all of TUs in .debug_info before CUs.
+  std::sort(CUVector.begin(), CUVector.end(),
+            [](const MapEntry &E1, const MapEntry &E2) -> bool {
+              return E1.second.Offset < E2.second.Offset;
+            });
+  // Writing out CU List <Offset, Size>
+  for (auto &CUInfo : CUVector) {
+    // Skipping TU for DWARF5 when they are not included in CU list.
+    if (!OriginalOffsets.count(CUInfo.first))
+      continue;
+    write64le(Buffer, CUInfo.second.Offset);
+    // Length encoded in CU doesn't contain first 4 bytes that encode length.
+    write64le(Buffer + 8, CUInfo.second.Length + 4);
+    Buffer += 16;
+  }
+
+  // Rewrite TU CU List, since abbrevs can be different.
+  // Entry example:
+  // 0: offset = 0x00000000, type_offset = 0x0000001e, type_signature =
+  // 0x418503b8111e9a7b Spec says " triplet, the first value is the CU offset,
+  // the second value is the type offset in the CU, and the third value is the
+  // type signature" Looking at what is being generated by gdb-add-index. The
+  // first entry is TU offset, second entry is offset from it, and third entry
+  // is the type signature.
+  if (TUListSize)
+    for (const GDBIndexTUEntry &Entry : getGDBIndexTUEntryVector()) {
+      write64le(Buffer, Entry.UnitOffset);
+      write64le(Buffer + 8, Entry.TypeDIERelativeOffset);
+      write64le(Buffer + 16, Entry.TypeHash);
+      Buffer += sizeof(GDBIndexTUEntry);
+    }
+
+  // Generate new address table.
+  for (const std::pair<const uint64_t, DebugAddressRangesVector> &CURangesPair :
+       ARangesSectionWriter.getCUAddressRanges()) {
+    const uint32_t CUIndex = OffsetToIndexMap[CURangesPair.first];
+    const DebugAddressRangesVector &Ranges = CURangesPair.second;
+    for (const DebugAddressRange &Range : Ranges) {
+      write64le(Buffer, Range.LowPC);
+      write64le(Buffer + 8, Range.HighPC);
+      write32le(Buffer + 16, CUIndex);
+      Buffer += 20;
+    }
+  }
+
+  const size_t TrailingSize =
+      GdbIndexContents.data() + GdbIndexContents.size() - Data;
+  assert(Buffer + TrailingSize == NewGdbIndexContents + NewGdbIndexSize &&
+         "size calculation error");
+
+  // Copy over the rest of the original data.
+  memcpy(Buffer, Data, TrailingSize);
+
+  // Register the new section.
+  BC.registerOrUpdateNoteSection(".gdb_index", NewGdbIndexContents,
+                                 NewGdbIndexSize);
+}
diff --git a/bolt/lib/Passes/VeneerElimination.cpp b/bolt/lib/Passes/VeneerElimination.cpp
index 0bec11128c7cea91b1a802e465476a2981be2bd7..87fe625e8c3b3e02a76139db4faaf00f089bd982 100644
--- a/bolt/lib/Passes/VeneerElimination.cpp
+++ b/bolt/lib/Passes/VeneerElimination.cpp
@@ -77,11 +77,8 @@ Error VeneerElimination::runOnFunctions(BinaryContext &BC) {
           continue;
 
         VeneerCallers++;
-        if (!BC.MIB->replaceBranchTarget(
-                Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get())) {
-          return createFatalBOLTError(
-              "BOLT-ERROR: updating veneer call destination failed\n");
-        }
+        BC.MIB->replaceBranchTarget(Instr, VeneerDestinations[TargetSymbol],
+                                    BC.Ctx.get());
       }
     }
   }
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index cdfca2b9871acfa6dee5adc57fedcb65557afb20..519f282a2351c2fb3653ce804b100bb223e9a12d 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -304,7 +304,7 @@ std::error_code BoltAddressTranslation::parse(raw_ostream &OS, StringRef Buf) {
 
   StringRef Name = Buf.slice(Offset, Offset + NameSz);
   Offset = alignTo(Offset + NameSz, 4);
-  if (Name.substr(0, 4) != "BOLT")
+  if (!Name.starts_with("BOLT"))
     return make_error_code(llvm::errc::io_error);
 
   Error Err(Error::success());
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 0ae9d3668b93bb87290a54b763e775a5234cfd6d..a74eda8e4a566eb2d3e1353f2f945f61eb8732a6 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -616,7 +616,7 @@ public:
     return getTargetAddend(Op.getExpr());
   }
 
-  bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
+  void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
                            MCContext *Ctx) const override {
     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
            "Invalid instruction");
@@ -638,7 +638,6 @@ public:
 
     *OI = MCOperand::createExpr(
         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
-    return true;
   }
 
   /// Matches indirect branch patterns in AArch64 related to a jump table (JT),
@@ -969,7 +968,7 @@ public:
     }
   }
 
-  bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
+  void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
                               MCContext *Ctx) const override {
     if (isTB(Inst) || isCB(Inst)) {
       Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode()));
@@ -984,7 +983,7 @@ public:
       LLVM_DEBUG(Inst.dump());
       llvm_unreachable("Unrecognized branch instruction");
     }
-    return replaceBranchTarget(Inst, TBB, Ctx);
+    replaceBranchTarget(Inst, TBB, Ctx);
   }
 
   int getPCRelEncodingSize(const MCInst &Inst) const override {
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 74f2f0aae91e667d12d6edff81f371c1e1044bfe..eb3f38a0b8f4a0d3301ba88cc8f597fd79f43939 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -151,14 +151,14 @@ public:
     }
   }
 
-  bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
+  void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
                               MCContext *Ctx) const override {
     auto Opcode = getInvertedBranchOpcode(Inst.getOpcode());
     Inst.setOpcode(Opcode);
-    return replaceBranchTarget(Inst, TBB, Ctx);
+    replaceBranchTarget(Inst, TBB, Ctx);
   }
 
-  bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
+  void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
                            MCContext *Ctx) const override {
     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
            "Invalid instruction");
@@ -170,7 +170,6 @@ public:
 
     Inst.getOperand(SymOpIndex) = MCOperand::createExpr(
         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
-    return true;
   }
 
   IndirectBranchType analyzeIndirectBranch(
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index a33a9dc8c013ce0b4c07591e64421b275f449b33..e350e701c7b7bae4bda206a3baa61976688b7c1b 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -2794,14 +2794,13 @@ public:
     Inst.addOperand(MCOperand::createImm(CC));
   }
 
-  bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
+  void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
                               MCContext *Ctx) const override {
     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
     Inst.getOperand(0) = MCOperand::createExpr(
         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
-    return true;
   }
 
   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
@@ -2844,13 +2843,12 @@ public:
     }
   }
 
-  bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
+  void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
                            MCContext *Ctx) const override {
     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
            "Invalid instruction");
     Inst.getOperand(0) = MCOperand::createExpr(
         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
-    return true;
   }
 
   MCPhysReg getX86R11() const override { return X86::R11; }
diff --git a/clang-tools-extra/clang-query/QueryParser.cpp b/clang-tools-extra/clang-query/QueryParser.cpp
index 1d0b7d9bc6fc84350f0497da542603661a4da3c8..97cb264a611af35f0dd833d6d34bbb016ef885a6 100644
--- a/clang-tools-extra/clang-query/QueryParser.cpp
+++ b/clang-tools-extra/clang-query/QueryParser.cpp
@@ -144,13 +144,11 @@ QueryRef QueryParser::endQuery(QueryRef Q) {
   StringRef Extra = Line;
   StringRef ExtraTrimmed = Extra.ltrim(" \t\v\f\r");
 
-  if ((!ExtraTrimmed.empty() && ExtraTrimmed[0] == '\n') ||
-      (ExtraTrimmed.size() >= 2 && ExtraTrimmed[0] == '\r' &&
-       ExtraTrimmed[1] == '\n'))
+  if (ExtraTrimmed.starts_with('\n') || ExtraTrimmed.starts_with("\r\n"))
     Q->RemainingContent = Extra;
   else {
     StringRef TrailingWord = lexWord();
-    if (!TrailingWord.empty() && TrailingWord.front() == '#') {
+    if (TrailingWord.starts_with('#')) {
       Line = Line.drop_until([](char c) { return c == '\n'; });
       Line = Line.drop_while([](char c) { return c == '\n'; });
       return endQuery(Q);
diff --git a/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
index 4dd3cb57e6dd12d83faef394caad692420cb4842..7a989b07119aad82e14a4e277c67f11502b3e078 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
@@ -48,12 +48,21 @@ AST_MATCHER(ImplicitCastExpr, isMultiLevelPointerConversion) {
   return SourcePtrLevel != TargetPtrLevel;
 }
 
+AST_MATCHER(QualType, isPointerType) {
+  const QualType Type =
+      Node.getCanonicalType().getNonReferenceType().getUnqualifiedType();
+
+  return !Type.isNull() && Type->isPointerType();
+}
+
 } // namespace
 
 void MultiLevelImplicitPointerConversionCheck::registerMatchers(
     MatchFinder *Finder) {
   Finder->addMatcher(
-      implicitCastExpr(hasCastKind(CK_BitCast), isMultiLevelPointerConversion())
+      implicitCastExpr(hasCastKind(CK_BitCast), isMultiLevelPointerConversion(),
+                       unless(hasParent(explicitCastExpr(
+                           hasDestinationType(isPointerType())))))
           .bind("expr"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
index 35e29b9a7d13670f2feb130aef1ef5cdef8d1119..1c1d3b836ea1b8cc96683ad9e9e47aeb280dbfef 100644
--- a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
@@ -41,9 +41,9 @@ add_clang_library(clangTidyMiscModule
   UnusedParametersCheck.cpp
   UnusedUsingDeclsCheck.cpp
   UseAnonymousNamespaceCheck.cpp
+  UseInternalLinkageCheck.cpp
 
   LINK_LIBS
-  clangAnalysis
   clangTidy
   clangTidyUtils
 
diff --git a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp
index fadfdc869d37b05403aa190b2577e1d4411e089d..cdb5e6b16069b7fe6b3e4fe56fbf09e8061d975c 100644
--- a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp
@@ -130,18 +130,15 @@ public:
         << FileName;
 
     const bool IsIncludePathValid =
-        std::all_of(Files.rbegin(), It, [](const Include &Elem) {
+        std::all_of(Files.rbegin(), It + 1, [](const Include &Elem) {
           return !Elem.Name.empty() && Elem.Loc.isValid();
         });
-
     if (!IsIncludePathValid)
       return;
 
-    auto CurrentIt = Files.rbegin();
-    do {
-      Check.diag(CurrentIt->Loc, "'%0' included from here", DiagnosticIDs::Note)
-          << CurrentIt->Name;
-    } while (CurrentIt++ != It);
+    for (const Include &I : llvm::make_range(Files.rbegin(), It + 1))
+      Check.diag(I.Loc, "'%0' included from here", DiagnosticIDs::Note)
+          << I.Name;
   }
 
   bool isFileIgnored(StringRef FileName) const {
diff --git a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
index d8a88324ee63e08b66bd84c3d71ecd9892283450..54bcebca7e186689bf0a0be910785f7250b4f059 100644
--- a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
@@ -31,6 +31,7 @@
 #include "UnusedParametersCheck.h"
 #include "UnusedUsingDeclsCheck.h"
 #include "UseAnonymousNamespaceCheck.h"
+#include "UseInternalLinkageCheck.h"
 
 namespace clang::tidy {
 namespace misc {
@@ -78,6 +79,8 @@ public:
         "misc-unused-using-decls");
     CheckFactories.registerCheck<UseAnonymousNamespaceCheck>(
         "misc-use-anonymous-namespace");
+    CheckFactories.registerCheck<UseInternalLinkageCheck>(
+        "misc-use-internal-linkage");
   }
 };
 
diff --git a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..70d0281df28fad816c7ec08205004f7a7e83d592
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
@@ -0,0 +1,95 @@
+//===--- UseInternalLinkageCheck.cpp - clang-tidy--------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UseInternalLinkageCheck.h"
+#include "../utils/FileExtensionsUtils.h"
+#include "clang/AST/Decl.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/ASTMatchersMacros.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/Specifiers.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::misc {
+
+namespace {
+
+AST_MATCHER(Decl, isFirstDecl) { return Node.isFirstDecl(); }
+
+static bool isInMainFile(SourceLocation L, SourceManager &SM,
+                         const FileExtensionsSet &HeaderFileExtensions) {
+  for (;;) {
+    if (utils::isSpellingLocInHeaderFile(L, SM, HeaderFileExtensions))
+      return false;
+    if (SM.isInMainFile(L))
+      return true;
+    // not in header file but not in main file
+    L = SM.getIncludeLoc(SM.getFileID(L));
+    if (L.isValid())
+      continue;
+    // Conservative about the unknown
+    return false;
+  }
+}
+
+AST_MATCHER_P(Decl, isAllRedeclsInMainFile, FileExtensionsSet,
+              HeaderFileExtensions) {
+  return llvm::all_of(Node.redecls(), [&](const Decl *D) {
+    return isInMainFile(D->getLocation(),
+                        Finder->getASTContext().getSourceManager(),
+                        HeaderFileExtensions);
+  });
+}
+
+AST_POLYMORPHIC_MATCHER(isExternStorageClass,
+                        AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
+                                                        VarDecl)) {
+  return Node.getStorageClass() == SC_Extern;
+}
+
+} // namespace
+
+void UseInternalLinkageCheck::registerMatchers(MatchFinder *Finder) {
+  auto Common =
+      allOf(isFirstDecl(), isAllRedeclsInMainFile(HeaderFileExtensions),
+            unless(anyOf(
+                // 1. internal linkage
+                isStaticStorageClass(), isInAnonymousNamespace(),
+                // 2. explicit external linkage
+                isExternStorageClass(), isExternC(),
+                // 3. template
+                isExplicitTemplateSpecialization(),
+                // 4. friend
+                hasAncestor(friendDecl()))));
+  Finder->addMatcher(
+      functionDecl(Common, unless(cxxMethodDecl()), unless(isMain()))
+          .bind("fn"),
+      this);
+  Finder->addMatcher(varDecl(Common, hasGlobalStorage()).bind("var"), this);
+}
+
+static constexpr StringRef Message =
+    "%0 %1 can be made static or moved into an anonymous namespace "
+    "to enforce internal linkage";
+
+void UseInternalLinkageCheck::check(const MatchFinder::MatchResult &Result) {
+  if (const auto *FD = Result.Nodes.getNodeAs<FunctionDecl>("fn")) {
+    diag(FD->getLocation(), Message) << "function" << FD;
+    return;
+  }
+  if (const auto *VD = Result.Nodes.getNodeAs<VarDecl>("var")) {
+    diag(VD->getLocation(), Message) << "variable" << VD;
+    return;
+  }
+  llvm_unreachable("");
+}
+
+} // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.h b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.h
new file mode 100644
index 0000000000000000000000000000000000000000..a3c1c33965903611bbb293ac1fa485a3dfbdd079
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.h
@@ -0,0 +1,38 @@
+//===--- UseInternalLinkageCheck.h - clang-tidy -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_USEINTERNALLINKAGECHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_USEINTERNALLINKAGECHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::misc {
+
+/// Detects variables and functions that can be marked as static or moved into
+/// an anonymous namespace to enforce internal linkage.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/misc/use-internal-linkage.html
+class UseInternalLinkageCheck : public ClangTidyCheck {
+public:
+  UseInternalLinkageCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context),
+        HeaderFileExtensions(Context->getHeaderFileExtensions()) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+
+private:
+  FileExtensionsSet HeaderFileExtensions;
+};
+
+} // namespace clang::tidy::misc
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_USEINTERNALLINKAGECHECK_H
diff --git a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
index ebc5338d0a7bfac8d03f4867adebbfc4e29f8d63..2a0cc403b726e8d4050ff63dfab3a00e15839fb5 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.cpp
@@ -32,6 +32,14 @@ static constexpr bool RestrictToPODTypesDefault = false;
 static constexpr char IgnoreMacrosName[] = "IgnoreMacros";
 static constexpr bool IgnoreMacrosDefault = true;
 
+static constexpr char StrictCStandardComplianceName[] =
+    "StrictCStandardCompliance";
+static constexpr bool StrictCStandardComplianceDefault = true;
+
+static constexpr char StrictCppStandardComplianceName[] =
+    "StrictCppStandardCompliance";
+static constexpr bool StrictCppStandardComplianceDefault = true;
+
 namespace {
 
 struct Designators {
@@ -97,7 +105,12 @@ UseDesignatedInitializersCheck::UseDesignatedInitializersCheck(
       RestrictToPODTypes(
           Options.get(RestrictToPODTypesName, RestrictToPODTypesDefault)),
       IgnoreMacros(
-          Options.getLocalOrGlobal(IgnoreMacrosName, IgnoreMacrosDefault)) {}
+          Options.getLocalOrGlobal(IgnoreMacrosName, IgnoreMacrosDefault)),
+      StrictCStandardCompliance(Options.get(StrictCStandardComplianceName,
+                                            StrictCStandardComplianceDefault)),
+      StrictCppStandardCompliance(
+          Options.get(StrictCppStandardComplianceName,
+                      StrictCppStandardComplianceDefault)) {}
 
 void UseDesignatedInitializersCheck::registerMatchers(MatchFinder *Finder) {
   const auto HasBaseWithFields =
@@ -179,6 +192,9 @@ void UseDesignatedInitializersCheck::storeOptions(
                 IgnoreSingleElementAggregates);
   Options.store(Opts, RestrictToPODTypesName, RestrictToPODTypes);
   Options.store(Opts, IgnoreMacrosName, IgnoreMacros);
+  Options.store(Opts, StrictCStandardComplianceName, StrictCStandardCompliance);
+  Options.store(Opts, StrictCppStandardComplianceName,
+                StrictCppStandardCompliance);
 }
 
 } // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.h b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.h
index 0a496f51b95762037634c764393c85f3d5841bbd..79095ade5037173e2810acf2559f4e5f641bdf8c 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/UseDesignatedInitializersCheck.h
@@ -29,10 +29,19 @@ public:
     return TK_IgnoreUnlessSpelledInSource;
   }
 
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus20 || LangOpts.C99 ||
+           (LangOpts.CPlusPlus && !StrictCppStandardCompliance) ||
+           (!LangOpts.CPlusPlus && !LangOpts.ObjC &&
+            !StrictCStandardCompliance);
+  }
+
 private:
   bool IgnoreSingleElementAggregates;
   bool RestrictToPODTypes;
   bool IgnoreMacros;
+  bool StrictCStandardCompliance;
+  bool StrictCppStandardCompliance;
 };
 
 } // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
index 9beb185cba929decf438eb430f7f773f6fdb9c32..61240fa4b0eb8e32aba4170793439b41ca0c3291 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
@@ -75,16 +75,16 @@ void recordRemoval(const DeclStmt &Stmt, ASTContext &Context,
   }
 }
 
-AST_MATCHER_FUNCTION_P(StatementMatcher, isConstRefReturningMethodCall,
+AST_MATCHER_FUNCTION_P(StatementMatcher,
+                       isRefReturningMethodCallWithConstOverloads,
                        std::vector<StringRef>, ExcludedContainerTypes) {
   // Match method call expressions where the `this` argument is only used as
-  // const, this will be checked in `check()` part. This returned const
-  // reference is highly likely to outlive the local const reference of the
-  // variable being declared. The assumption is that the const reference being
-  // returned either points to a global static variable or to a member of the
-  // called object.
+  // const, this will be checked in `check()` part. This returned reference is
+  // highly likely to outlive the local const reference of the variable being
+  // declared. The assumption is that the reference being returned either points
+  // to a global static variable or to a member of the called object.
   const auto MethodDecl =
-      cxxMethodDecl(returns(hasCanonicalType(matchers::isReferenceToConst())))
+      cxxMethodDecl(returns(hasCanonicalType(referenceType())))
           .bind(MethodDeclId);
   const auto ReceiverExpr =
       ignoringParenImpCasts(declRefExpr(to(varDecl().bind(ObjectArgId))));
@@ -121,7 +121,7 @@ AST_MATCHER_FUNCTION_P(StatementMatcher, initializerReturnsReferenceToConst,
       declRefExpr(to(varDecl(hasLocalStorage()).bind(OldVarDeclId)));
   return expr(
       anyOf(isConstRefReturningFunctionCall(),
-            isConstRefReturningMethodCall(ExcludedContainerTypes),
+            isRefReturningMethodCallWithConstOverloads(ExcludedContainerTypes),
             ignoringImpCasts(OldVarDeclRef),
             ignoringImpCasts(unaryOperator(hasOperatorName("&"),
                                            hasUnaryOperand(OldVarDeclRef)))));
@@ -259,10 +259,11 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
         .bind("blockStmt");
   };
 
-  Finder->addMatcher(LocalVarCopiedFrom(anyOf(isConstRefReturningFunctionCall(),
-                                              isConstRefReturningMethodCall(
-                                                  ExcludedContainerTypes))),
-                     this);
+  Finder->addMatcher(
+      LocalVarCopiedFrom(anyOf(
+          isConstRefReturningFunctionCall(),
+          isRefReturningMethodCallWithConstOverloads(ExcludedContainerTypes))),
+      this);
 
   Finder->addMatcher(LocalVarCopiedFrom(declRefExpr(
                          to(varDecl(hasLocalStorage()).bind(OldVarDeclId)))),
diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
index bbc1b47b97ae6143c01cf5bd9afce7a18e3fc008..bf7a847dff103cd97679039d3121a07a14560b72 100644
--- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
@@ -96,9 +96,14 @@ AST_MATCHER(QualType, isIntegralType) {
 
 AST_MATCHER_P(UserDefinedLiteral, hasLiteral,
               clang::ast_matchers::internal::Matcher<Expr>, InnerMatcher) {
-  if (const Expr *CookedLiteral = Node.getCookedLiteral()) {
+  const UserDefinedLiteral::LiteralOperatorKind LOK =
+      Node.getLiteralOperatorKind();
+  if (LOK == UserDefinedLiteral::LOK_Template ||
+      LOK == UserDefinedLiteral::LOK_Raw)
+    return false;
+
+  if (const Expr *CookedLiteral = Node.getCookedLiteral())
     return InnerMatcher.matches(*CookedLiteral, Finder, Builder);
-  }
   return false;
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
index 28f5eada6d825af19876a096d90f6ee95181d6aa..aa115cd450c4f619f1035e601e83a2910d295c67 100644
--- a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
@@ -279,6 +279,9 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
       hasParent(callExpr()),
       hasSourceExpression(binaryOperator(hasAnyOperatorName("==", "!="))));
 
+  auto IsInCompilerGeneratedFunction = hasAncestor(namedDecl(anyOf(
+      isImplicit(), functionDecl(isDefaulted()), functionTemplateDecl())));
+
   Finder->addMatcher(
       traverse(TK_AsIs,
                implicitCastExpr(
@@ -299,7 +302,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
                    // additional parens in replacement.
                    optionally(hasParent(stmt().bind("parentStmt"))),
                    unless(isInTemplateInstantiation()),
-                   unless(hasAncestor(functionTemplateDecl())))
+                   unless(IsInCompilerGeneratedFunction))
                    .bind("implicitCastToBool")),
       this);
 
@@ -331,7 +334,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
               anyOf(hasParent(implicitCastExpr().bind("furtherImplicitCast")),
                     anything()),
               unless(isInTemplateInstantiation()),
-              unless(hasAncestor(functionTemplateDecl())))),
+              unless(IsInCompilerGeneratedFunction))),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp b/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
index 65fd296094915bbddf4e114e3ee96f8d5efb5f50..64ce94e3fc1db60f4f13fdab79ec41ecd2acd9fe 100644
--- a/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
@@ -57,7 +57,8 @@ static void addParantheses(const BinaryOperator *BinOp,
   int Precedence1 = getPrecedence(BinOp);
   int Precedence2 = getPrecedence(ParentBinOp);
 
-  if (ParentBinOp != nullptr && Precedence1 != Precedence2) {
+  if (ParentBinOp != nullptr && Precedence1 != Precedence2 && Precedence1 > 0 &&
+      Precedence2 > 0) {
     const clang::SourceLocation StartLoc = BinOp->getBeginLoc();
     const clang::SourceLocation EndLoc =
         clang::Lexer::getLocForEndOfToken(BinOp->getEndLoc(), 0, SM, LangOpts);
diff --git a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
index a48e45e13568134d44a5a080757c5838407823a6..a6062ccf42230bf304c19621249b95448248a63f 100644
--- a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
+++ b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
@@ -36,6 +36,116 @@ void extractNodesByIdTo(ArrayRef<BoundNodes> Matches, StringRef ID,
     Nodes.insert(Match.getNodeAs<Node>(ID));
 }
 
+// Returns true if both types refer to the same type,
+// ignoring the const-qualifier.
+bool isSameTypeIgnoringConst(QualType A, QualType B) {
+  A = A.getCanonicalType();
+  B = B.getCanonicalType();
+  A.addConst();
+  B.addConst();
+  return A == B;
+}
+
+// Returns true if `D` and `O` have the same parameter types.
+bool hasSameParameterTypes(const CXXMethodDecl &D, const CXXMethodDecl &O) {
+  if (D.getNumParams() != O.getNumParams())
+    return false;
+  for (int I = 0, E = D.getNumParams(); I < E; ++I) {
+    if (!isSameTypeIgnoringConst(D.getParamDecl(I)->getType(),
+                                 O.getParamDecl(I)->getType()))
+      return false;
+  }
+  return true;
+}
+
+// If `D` has a const-qualified overload with otherwise identical
+// ref-qualifiers and parameter types, returns that overload.
+const CXXMethodDecl *findConstOverload(const CXXMethodDecl &D) {
+  assert(!D.isConst());
+
+  DeclContext::lookup_result LookupResult =
+      D.getParent()->lookup(D.getNameInfo().getName());
+  if (LookupResult.isSingleResult()) {
+    // No overload.
+    return nullptr;
+  }
+  for (const Decl *Overload : LookupResult) {
+    const auto *O = dyn_cast<CXXMethodDecl>(Overload);
+    if (O && !O->isDeleted() && O->isConst() &&
+        O->getRefQualifier() == D.getRefQualifier() &&
+        hasSameParameterTypes(D, *O))
+      return O;
+  }
+  return nullptr;
+}
+
+// Returns true if both types are pointers or reference to the same type,
+// ignoring the const-qualifier.
+bool pointsToSameTypeIgnoringConst(QualType A, QualType B) {
+  assert(A->isPointerType() || A->isReferenceType());
+  assert(B->isPointerType() || B->isReferenceType());
+  return isSameTypeIgnoringConst(A->getPointeeType(), B->getPointeeType());
+}
+
+// Return true if non-const member function `M` likely does not mutate `*this`.
+//
+// Note that if the member call selects a method/operator `f` that
+// is not const-qualified, then we also consider that the object is
+// not mutated if:
+//  - (A) there is a const-qualified overload `cf` of `f` that has
+//  the
+//    same ref-qualifiers;
+//  - (B) * `f` returns a value, or
+//        * if `f` returns a `T&`, `cf` returns a `const T&` (up to
+//          possible aliases such as `reference` and
+//          `const_reference`), or
+//        * if `f` returns a `T*`, `cf` returns a `const T*` (up to
+//          possible aliases).
+//  - (C) the result of the call is not mutated.
+//
+// The assumption that `cf` has the same semantics as `f`.
+// For example:
+//   - In `std::vector<T> v; const T t = v[...];`, we consider that
+//     expression `v[...]` does not mutate `v` as
+//    `T& std::vector<T>::operator[]` has a const overload
+//     `const T& std::vector<T>::operator[] const`, and the
+//     result expression of type `T&` is only used as a `const T&`;
+//   - In `std::map<K, V> m; V v = m.at(...);`, we consider
+//     `m.at(...)` to be an immutable access for the same reason.
+// However:
+//   - In `std::map<K, V> m; const V v = m[...];`, We consider that
+//     `m[...]` mutates `m` as `V& std::map<K, V>::operator[]` does
+//     not have a const overload.
+//   - In `std::vector<T> v; T& t = v[...];`, we consider that
+//     expression `v[...]` mutates `v` as the result is kept as a
+//     mutable reference.
+//
+// This function checks (A) ad (B), but the caller should make sure that the
+// object is not mutated through the return value.
+bool isLikelyShallowConst(const CXXMethodDecl &M) {
+  assert(!M.isConst());
+  // The method can mutate our variable.
+
+  // (A)
+  const CXXMethodDecl *ConstOverload = findConstOverload(M);
+  if (ConstOverload == nullptr) {
+    return false;
+  }
+
+  // (B)
+  const QualType CallTy = M.getReturnType().getCanonicalType();
+  const QualType OverloadTy = ConstOverload->getReturnType().getCanonicalType();
+  if (CallTy->isReferenceType()) {
+    return OverloadTy->isReferenceType() &&
+           pointsToSameTypeIgnoringConst(CallTy, OverloadTy);
+  }
+  if (CallTy->isPointerType()) {
+    return OverloadTy->isPointerType() &&
+           pointsToSameTypeIgnoringConst(CallTy, OverloadTy);
+  }
+  return isSameTypeIgnoringConst(CallTy, OverloadTy);
+}
+
 // A matcher that matches DeclRefExprs that are used in ways such that the
 // underlying declaration is not modified.
 // If the declaration is of pointer type, `Indirections` specifies the level
@@ -54,16 +164,15 @@ void extractNodesByIdTo(ArrayRef<BoundNodes> Matches, StringRef ID,
 //  matches (A).
 //
 AST_MATCHER_P(DeclRefExpr, doesNotMutateObject, int, Indirections) {
-  // We walk up the parents of the DeclRefExpr recursively until we end up on a
-  // parent that cannot modify the underlying object. There are a few kinds of
-  // expressions:
-  //  - Those that cannot be used to mutate the underlying object. We can stop
+  // We walk up the parents of the DeclRefExpr recursively. There are a few
+  // kinds of expressions:
+  //  - Those that cannot be used to mutate the underlying variable. We can stop
   //    recursion there.
-  //  - Those that can be used to mutate the underlying object in analyzable
+  //  - Those that can be used to mutate the underlying variable in analyzable
   //    ways (such as taking the address or accessing a subobject). We have to
   //    examine the parents.
   //  - Those that we don't know how to analyze. In that case we stop there and
-  //    we assume that they can mutate the underlying expression.
+  //    we assume that they can modify the expression.
 
   struct StackEntry {
     StackEntry(const Expr *E, int Indirections)
@@ -90,7 +199,7 @@ AST_MATCHER_P(DeclRefExpr, doesNotMutateObject, int, Indirections) {
       assert(Ty->isPointerType());
       Ty = Ty->getPointeeType().getCanonicalType();
     }
-    if (Ty.isConstQualified())
+    if (Ty->isVoidType() || Ty.isConstQualified())
       continue;
 
     // Otherwise we have to look at the parents to see how the expression is
@@ -159,11 +268,56 @@ AST_MATCHER_P(DeclRefExpr, doesNotMutateObject, int, Indirections) {
             // The method call cannot mutate our variable.
             continue;
           }
+          if (isLikelyShallowConst(*Method)) {
+            // We still have to check that the object is not modified through
+            // the method's return value (C).
+            const auto MemberParents = Ctx.getParents(*Member);
+            assert(MemberParents.size() == 1);
+            const auto *Call = MemberParents[0].get<CallExpr>();
+            // If `o` is an object of class type and `f` is a member function,
+            // then `o.f` has to be used as part of a call expression.
+            assert(Call != nullptr && "member function has to be called");
+            Stack.emplace_back(
+                Call,
+                Method->getReturnType().getCanonicalType()->isPointerType()
+                    ? 1
+                    : 0);
+            continue;
+          }
           return false;
         }
         Stack.emplace_back(Member, 0);
         continue;
       }
+      if (const auto *const OpCall = dyn_cast<CXXOperatorCallExpr>(P)) {
+        // Operator calls have function call syntax. The `*this` parameter
+        // is the first parameter.
+        if (OpCall->getNumArgs() == 0 || OpCall->getArg(0) != Entry.E) {
+          return false;
+        }
+        const auto *const Method =
+            dyn_cast<CXXMethodDecl>(OpCall->getDirectCallee());
+
+        if (Method == nullptr) {
+          // This is not a member operator. Typically, a friend operator. These
+          // are handled like function calls.
+          return false;
+        }
+
+        if (Method->isConst() || Method->isStatic()) {
+          continue;
+        }
+        if (isLikelyShallowConst(*Method)) {
+          // We still have to check that the object is not modified through
+          // the operator's return value (C).
+          Stack.emplace_back(
+              OpCall,
+              Method->getReturnType().getCanonicalType()->isPointerType() ? 1
+                                                                          : 0);
+          continue;
+        }
+        return false;
+      }
 
       if (const auto *const Op = dyn_cast<UnaryOperator>(P)) {
         switch (Op->getOpcode()) {
diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index 5244a4e893769ed373e310ff1795ecf4419df7e1..55f16b7085a6fec6508ea23bcb6c6220c14f0b61 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -454,7 +454,7 @@ private:
       if (!MacroName.isValid() || !MacroName.isFileID())
         continue;
       // All conditions satisfied, add the macro.
-      if (auto *Tok = AST.getTokens().spelledTokenAt(MacroName))
+      if (auto *Tok = AST.getTokens().spelledTokenContaining(MacroName))
         CurParent = &CurParent->inMacro(
             *Tok, SM, AST.getTokens().expansionStartingAt(Tok));
     }
diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index 01b47679790f1d52633b8bb8b49aeca13147a060..dc5b7ec95db5ff1617bf1d4502df5563b060e637 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -303,7 +303,7 @@ collectMacroReferences(ParsedAST &AST) {
   for (const auto &[_, Refs] : AST.getMacros().MacroRefs) {
     for (const auto &Ref : Refs) {
       auto Loc = SM.getComposedLoc(SM.getMainFileID(), Ref.StartOffset);
-      const auto *Tok = AST.getTokens().spelledTokenAt(Loc);
+      const auto *Tok = AST.getTokens().spelledTokenContaining(Loc);
       if (!Tok)
         continue;
       auto Macro = locateMacroAt(*Tok, PP);
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index eb025f21f3616126ee7e0a22d0019f95bdae0073..a366f1331c2d3d999982eb38b0bf7a9e3dffccb8 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -447,11 +447,10 @@ public:
     if (!RLoc.isValid())
       return;
 
-    const auto *RTok = TB.spelledTokenAt(RLoc);
-    // Handle `>>`. RLoc is always pointing at the right location, just change
-    // the end to be offset by 1.
-    // We'll either point at the beginning of `>>`, hence get a proper spelled
-    // or point in the middle of `>>` hence get no spelled tok.
+    const auto *RTok = TB.spelledTokenContaining(RLoc);
+    // Handle `>>`. RLoc is either part of `>>` or a spelled token on its own
+    // `>`. If it's the former, slice to have length of 1, if latter use the
+    // token as-is.
     if (!RTok || RTok->kind() == tok::greatergreater) {
       Position Begin = sourceLocToPosition(SourceMgr, RLoc);
       Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1));
@@ -577,7 +576,7 @@ private:
       return std::nullopt;
     // We might have offsets in the main file that don't correspond to any
     // spelled tokens.
-    const auto *Tok = TB.spelledTokenAt(Loc);
+    const auto *Tok = TB.spelledTokenContaining(Loc);
     if (!Tok)
       return std::nullopt;
     return halfOpenToRange(SourceMgr,
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index cd909266489a850b54322ca21edd191a3519c3a5..f94cadeffaa298ab4bd5d6a583d604913dc34116 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -844,7 +844,7 @@ std::vector<DocumentLink> getDocumentLinks(ParsedAST &AST) {
     if (Inc.Resolved.empty())
       continue;
     auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset);
-    const auto *HashTok = AST.getTokens().spelledTokenAt(HashLoc);
+    const auto *HashTok = AST.getTokens().spelledTokenContaining(HashLoc);
     assert(HashTok && "got inclusion at wrong offset");
     const auto *IncludeTok = std::next(HashTok);
     const auto *FileTok = std::next(IncludeTok);
@@ -938,7 +938,7 @@ public:
     CollectorOpts.CollectMainFileSymbols = true;
     for (SourceLocation L : Locs) {
       L = SM.getFileLoc(L);
-      if (const auto *Tok = TB.spelledTokenAt(L))
+      if (const auto *Tok = TB.spelledTokenContaining(L))
         References.push_back(
             {*Tok, Roles,
              SymbolCollector::getRefContainer(ASTNode.Parent, CollectorOpts)});
@@ -1216,7 +1216,7 @@ DocumentHighlight toHighlight(const ReferenceFinder::Reference &Ref,
 std::optional<DocumentHighlight> toHighlight(SourceLocation Loc,
                                              const syntax::TokenBuffer &TB) {
   Loc = TB.sourceManager().getFileLoc(Loc);
-  if (const auto *Tok = TB.spelledTokenAt(Loc)) {
+  if (const auto *Tok = TB.spelledTokenContaining(Loc)) {
     DocumentHighlight Result;
     Result.range = halfOpenToRange(
         TB.sourceManager(),
@@ -1353,7 +1353,8 @@ maybeFindIncludeReferences(ParsedAST &AST, Position Pos,
           Loc = SM.getIncludeLoc(SM.getFileID(Loc));
 
         ReferencesResult::Reference Result;
-        const auto *Token = AST.getTokens().spelledTokenAt(Loc);
+        const auto *Token = AST.getTokens().spelledTokenContaining(Loc);
+        assert(Token && "references expected token here");
         Result.Loc.range = Range{sourceLocToPosition(SM, Token->location()),
                                  sourceLocToPosition(SM, Token->endLocation())};
         Result.Loc.uri = URIMainFile;
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index c0fc4453a3fccc61bb8abe75c7abf139b98e10ca..c85e13dbdfe97f00bb8474e7504ebb3faa2c7acc 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -748,7 +748,7 @@ std::vector<SymbolRange> collectRenameIdentifierRanges(
 clangd::Range tokenRangeForLoc(ParsedAST &AST, SourceLocation TokLoc,
                                const SourceManager &SM,
                                const LangOptions &LangOpts) {
-  const auto *Token = AST.getTokens().spelledTokenAt(TokLoc);
+  const auto *Token = AST.getTokens().spelledTokenContaining(TokLoc);
   assert(Token && "rename expects spelled tokens");
   clangd::Range Result;
   Result.start = sourceLocToPosition(SM, Token->location());
diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
index 6420516e78557672aae2301d3b0c06fca62ff7e0..16a2f9448b1ecf4d48548d87ea415394955f8cc5 100644
--- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
@@ -417,7 +417,7 @@ TEST(PreamblePatchTest, LocateMacroAtWorks) {
     ASSERT_TRUE(AST);
 
     const auto &SM = AST->getSourceManager();
-    auto *MacroTok = AST->getTokens().spelledTokenAt(
+    auto *MacroTok = AST->getTokens().spelledTokenContaining(
         SM.getComposedLoc(SM.getMainFileID(), Modified.point("use")));
     ASSERT_TRUE(MacroTok);
 
@@ -441,7 +441,7 @@ TEST(PreamblePatchTest, LocateMacroAtDeletion) {
     ASSERT_TRUE(AST);
 
     const auto &SM = AST->getSourceManager();
-    auto *MacroTok = AST->getTokens().spelledTokenAt(
+    auto *MacroTok = AST->getTokens().spelledTokenContaining(
         SM.getComposedLoc(SM.getMainFileID(), Modified.point()));
     ASSERT_TRUE(MacroTok);
 
@@ -512,9 +512,10 @@ TEST(PreamblePatchTest, RefsToMacros) {
       ExpectedLocations.push_back(referenceRangeIs(R));
 
     for (const auto &P : Modified.points()) {
-      auto *MacroTok = AST->getTokens().spelledTokenAt(SM.getComposedLoc(
-          SM.getMainFileID(),
-          llvm::cantFail(positionToOffset(Modified.code(), P))));
+      auto *MacroTok =
+          AST->getTokens().spelledTokenContaining(SM.getComposedLoc(
+              SM.getMainFileID(),
+              llvm::cantFail(positionToOffset(Modified.code(), P))));
       ASSERT_TRUE(MacroTok);
       EXPECT_THAT(findReferences(*AST, P, 0).References,
                   testing::ElementsAreArray(ExpectedLocations));
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index f53cbf01b7992c8d59daa3cd09492ddfe2324157..cbceb9a343f87ca2d4a502cb5f987986dfc8a4b6 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -2173,6 +2173,11 @@ TEST(FindReferences, WithinAST) {
         using $def[[MyTypeD^ef]] = int;
         enum MyEnum : $(MyEnum)[[MyTy^peDef]] { };
       )cpp",
+      // UDL
+      R"cpp(
+        bool $decl[[operator]]"" _u^dl(unsigned long long value);
+        bool x = $(x)[[1_udl]];
+      )cpp",
   };
   for (const char *Test : Tests)
     checkFindRefs(Test);
@@ -2358,7 +2363,13 @@ TEST(FindReferences, UsedSymbolsFromInclude) {
 
       R"cpp([[#in^clude <vector>]]
         std::[[vector]]<int> vec;
-      )cpp"};
+      )cpp",
+
+      R"cpp(
+        [[#include ^"udl_header.h"]]
+        auto x = [[1_b]];
+      )cpp",
+  };
   for (const char *Test : Tests) {
     Annotations T(Test);
     auto TU = TestTU::withCode(T.code());
@@ -2375,6 +2386,9 @@ TEST(FindReferences, UsedSymbolsFromInclude) {
         class vector{};
       }
     )cpp");
+    TU.AdditionalFiles["udl_header.h"] = guard(R"cpp(
+      bool operator"" _b(unsigned long long value);
+    )cpp");
     TU.ExtraArgs.push_back("-isystem" + testPath("system"));
 
     auto AST = TU.build();
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 6947cf06f6e56a075c91b8dd4ff4e01bb8dd1d0f..0c0c10605a83070737f77c5a6b2fe3c263b0f93f 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -148,6 +148,12 @@ New checks
   to reading out-of-bounds data due to inadequate or incorrect string null
   termination.
 
+- New :doc:`misc-use-internal-linkage
+  <clang-tidy/checks/misc/use-internal-linkage>` check.
+
+  Detects variables and functions that can be marked as static or moved into
+  an anonymous namespace to enforce internal linkage.
+
 - New :doc:`modernize-min-max-use-initializer-list
   <clang-tidy/checks/modernize/min-max-use-initializer-list>` check.
 
@@ -218,6 +224,10 @@ Changes in existing checks
   check by ignoring ``__func__`` macro in lambda captures, initializers of
   default parameters and nested function declarations.
 
+- Improved :doc:`bugprone-multi-level-implicit-pointer-conversion
+  <clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion>` check
+  by ignoring implicit pointer conversions that are part of a cast expression.
+
 - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion
   <clang-tidy/checks/bugprone/non-zero-enum-to-bool-conversion>` check by
   eliminating false positives resulting from direct usage of bitwise operators
@@ -317,6 +327,10 @@ Changes in existing checks
   Additionally, the option `UseHeaderFileExtensions` is removed, so that the
   check uses the `HeaderFileExtensions` option unconditionally.
 
+- Improved :doc:`misc-header-include-cycle
+  <clang-tidy/checks/misc/header-include-cycle>` check by avoiding crash for self
+  include cycles.
+
 - Improved :doc:`misc-unused-using-decls
   <clang-tidy/checks/misc/unused-using-decls>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
@@ -362,8 +376,10 @@ Changes in existing checks
 - Improved :doc:`performance-unnecessary-copy-initialization
   <clang-tidy/checks/performance/unnecessary-copy-initialization>` check by
   detecting more cases of constant access. In particular, pointers can be
-  analyzed, se the check now handles the common patterns
+  analyzed, so the check now handles the common patterns
   `const auto e = (*vector_ptr)[i]` and `const auto e = vector_ptr->at(i);`.
+  Calls to mutable function where there exists a `const` overload are also
+  handled.
 
 - Improved :doc:`readability-avoid-return-with-void-value
   <clang-tidy/checks/readability/avoid-return-with-void-value>` check by adding
@@ -376,6 +392,7 @@ Changes in existing checks
 - Improved :doc:`readability-container-size-empty
   <clang-tidy/checks/readability/container-size-empty>` check to prevent false
   positives when utilizing ``size`` or ``length`` methods that accept parameter.
+  Fixed crash when facing template user defined literals.
 
 - Improved :doc:`readability-duplicate-include
   <clang-tidy/checks/readability/duplicate-include>` check by excluding include
@@ -397,7 +414,8 @@ Changes in existing checks
   valid fix suggestions for ``static_cast`` without a preceding space and
   fixed problem with duplicate parentheses in double implicit casts. Corrected
   the fix suggestions for C23 and later by using C-style casts instead of
-  ``static_cast``.
+  ``static_cast``. Fixed false positives in C++20 spaceship operator by ignoring
+  casts in implicit and defaulted functions.
 
 - Improved :doc:`readability-redundant-inline-specifier
   <clang-tidy/checks/readability/redundant-inline-specifier>` check to properly
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 85e4f0352ac22b2b4e23cc36eb6b14814b48a08b..87d3db20f768473b8266f18ce95664ca15f955e9 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -267,6 +267,7 @@ Clang-Tidy Checks
    :doc:`misc-unused-parameters <misc/unused-parameters>`, "Yes"
    :doc:`misc-unused-using-decls <misc/unused-using-decls>`, "Yes"
    :doc:`misc-use-anonymous-namespace <misc/use-anonymous-namespace>`,
+   :doc:`misc-use-internal-linkage <misc/use-internal-linkage>`,
    :doc:`modernize-avoid-bind <modernize/avoid-bind>`, "Yes"
    :doc:`modernize-avoid-c-arrays <modernize/avoid-c-arrays>`,
    :doc:`modernize-concat-nested-namespaces <modernize/concat-nested-namespaces>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e8e43a1fb3d63249bcfa84b88fd1fd7f34717bf2
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst
@@ -0,0 +1,27 @@
+.. title:: clang-tidy - misc-use-internal-linkage
+
+misc-use-internal-linkage
+=========================
+
+Detects variables and functions that can be marked as static or moved into
+an anonymous namespace to enforce internal linkage.
+
+Static functions and variables are scoped to a single file. Marking functions
+and variables as static helps to better remove dead code. In addition, it gives
+the compiler more information and allows for more aggressive optimizations.
+
+Example:
+
+.. code-block:: c++
+
+  int v1; // can be marked as static
+
+  void fn1(); // can be marked as static
+
+  namespace {
+    // already in anonymous namespace
+    int v2;
+    void fn2();
+  }
+  // already declared as extern
+  extern int v2;
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-designated-initializers.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-designated-initializers.rst
index 22f50980baade6a4b4a7fa80aed5ce819fbe869b..f101cfc6f3a2b3954d2d0a73230ae176c9a86bcc 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-designated-initializers.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-designated-initializers.rst
@@ -37,7 +37,7 @@ declaration of ``S``.
 
 Even when compiling in a language version older than C++20, depending on your
 compiler, designated initializers are potentially supported. Therefore, the
-check is not restricted to C++20 and newer versions. Check out the options
+check is by default restricted to C99/C++20 and above. Check out the options
 ``-Wc99-designator`` to get support for mixed designators in initializer list in
 C and ``-Wc++20-designator`` for support of designated initializers in older C++
 language modes.
@@ -60,3 +60,13 @@ Options
     The value `true` specifies that only Plain Old Data (POD) types shall be
     checked. This makes the check applicable to even older C++ standards. The
     default value is `false`.
+
+.. option:: StrictCStandardCompliance
+
+   When set to `false`, the check will not restrict itself to C99 and above.
+   The default value is `true`.
+
+.. option:: StrictCppStandardCompliance
+
+   When set to `false`, the check will not restrict itself to C++20 and above.
+   The default value is `true`.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.cpp
index 7a56242e4202d61d9a2f3b0f52c336fb77b6dda7..6868f9e5909088c68fb929619b0b5f686b794ca3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.cpp
@@ -63,3 +63,15 @@ void test()
 
   takeSecondLevelVoidPtr(getSecondLevelVoidPtr());
 }
+
+namespace PR93959 {
+  void free(void*);
+
+  void test() {
+    char **p = nullptr;
+    free(p);
+    // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: multilevel pointer conversion from 'char **' to 'void *', please use explicit cast [bugprone-multi-level-implicit-pointer-conversion]
+    free((void *)p);
+    free(static_cast<void *>(p));
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func.h b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func.h
new file mode 100644
index 0000000000000000000000000000000000000000..0f2b576a126c4a422657c7063a81d3362e8f38d4
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func.h
@@ -0,0 +1,5 @@
+#pragma once
+
+void func_header();
+
+#include "func_h.inc"
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_cpp.inc b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_cpp.inc
new file mode 100644
index 0000000000000000000000000000000000000000..97e026f0116e99542d83e9fccda74bf4262469ef
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_cpp.inc
@@ -0,0 +1 @@
+void func_cpp_inc();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_h.inc b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_h.inc
new file mode 100644
index 0000000000000000000000000000000000000000..1130f710edd7c3d5090daed8e148fc3e7a6e290d
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/func_h.inc
@@ -0,0 +1 @@
+void func_h_inc();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/var.h b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/var.h
new file mode 100644
index 0000000000000000000000000000000000000000..37e4cfbafff1460fd214cebc89357193f4e9ade7
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/use-internal-linkage/var.h
@@ -0,0 +1,3 @@
+#pragma once
+
+extern int gloabl_header;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..245dd0a65a8b42bf696e20586783de572a865bac
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp
@@ -0,0 +1,3 @@
+// RUN: not clang-tidy %s -checks='-*,misc-header-include-cycle'
+
+#include "header-include-cycle.self.cpp"
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c6c513fe0b0c06329029499bd0d3729d18ffb1db
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp
@@ -0,0 +1,37 @@
+// RUN: %check_clang_tidy %s misc-use-internal-linkage %t -- -- -I%S/Inputs/use-internal-linkage
+
+#include "func.h"
+
+void func() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: function 'func'
+
+template<class T>
+void func_template() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: function 'func_template'
+
+void func_cpp_inc();
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: function 'func_cpp_inc'
+
+#include "func_cpp.inc"
+
+void func_h_inc();
+
+struct S {
+  void method();
+};
+void S::method() {}
+
+void func_header();
+extern void func_extern();
+static void func_static();
+namespace {
+void func_anonymous_ns();
+} // namespace
+
+int main(int argc, const char*argv[]) {}
+
+extern "C" {
+void func_extern_c_1() {}
+}
+
+extern "C" void func_extern_c_2() {}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-var.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-var.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bd5ef5431de6ccdd23bb9851f37b8f1af20c1bea
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-var.cpp
@@ -0,0 +1,40 @@
+// RUN: %check_clang_tidy %s misc-use-internal-linkage %t -- -- -I%S/Inputs/use-internal-linkage
+
+#include "var.h"
+
+int global;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: variable 'global'
+
+template<class T>
+T global_template;
+// CHECK-MESSAGES: :[[@LINE-1]]:3: warning: variable 'global_template'
+
+int gloabl_header;
+
+extern int global_extern;
+
+static int global_static;
+
+namespace {
+static int global_anonymous_ns;
+namespace NS {
+static int global_anonymous_ns;
+}
+}
+
+static void f(int para) {
+  int local;
+  static int local_static;
+}
+
+struct S {
+  int m1;
+  static int m2;
+};
+int S::m2;
+
+extern "C" {
+int global_in_extern_c_1;
+}
+
+extern "C" int global_in_extern_c_2;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
index 7e5c26e3f4404a1759c225a9a50501718d7c19bc..9b769ad0be23cab8e399bea24797f95b45973a46 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-designated-initializers.cpp
@@ -1,13 +1,13 @@
-// RUN: %check_clang_tidy -std=c++17 %s modernize-use-designated-initializers %t \
+// RUN: %check_clang_tidy -std=c++20 %s modernize-use-designated-initializers %t \
 // RUN:     -- \
 // RUN:     -- -fno-delayed-template-parsing
-// RUN: %check_clang_tidy -check-suffixes=,SINGLE-ELEMENT -std=c++17 %s modernize-use-designated-initializers %t \
+// RUN: %check_clang_tidy -check-suffixes=,SINGLE-ELEMENT -std=c++20 %s modernize-use-designated-initializers %t \
 // RUN:     -- -config="{CheckOptions: {modernize-use-designated-initializers.IgnoreSingleElementAggregates: false}}" \
 // RUN:     -- -fno-delayed-template-parsing
-// RUN: %check_clang_tidy -check-suffixes=POD -std=c++17 %s modernize-use-designated-initializers %t \
+// RUN: %check_clang_tidy -check-suffixes=POD -std=c++20 %s modernize-use-designated-initializers %t \
 // RUN:     -- -config="{CheckOptions: {modernize-use-designated-initializers.RestrictToPODTypes: true}}" \
 // RUN:     -- -fno-delayed-template-parsing
-// RUN: %check_clang_tidy -check-suffixes=,MACROS -std=c++17 %s modernize-use-designated-initializers %t \
+// RUN: %check_clang_tidy -check-suffixes=,MACROS -std=c++20 %s modernize-use-designated-initializers %t \
 // RUN:     -- -config="{CheckOptions: {modernize-use-designated-initializers.IgnoreMacros: false}}" \
 // RUN:     -- -fno-delayed-template-parsing
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/unnecessary-copy-initialization.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/unnecessary-copy-initialization.cpp
index 92625cc1332e286296075f4ef2e2b7654cb308ed..f259552dc8f1d89fd1d36eb9e17876fb08d3c68b 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/unnecessary-copy-initialization.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/unnecessary-copy-initialization.cpp
@@ -32,6 +32,9 @@ struct ExpensiveToCopyType {
 
 template <typename T>
 struct Container {
+  using reference = T&;
+  using const_reference = const T&;
+
   bool empty() const;
   const T& operator[](int) const;
   const T& operator[](int);
@@ -42,8 +45,8 @@ struct Container {
   void nonConstMethod();
   bool constMethod() const;
 
-  const T& at(int) const;
-  T& at(int);
+  reference at(int) const;
+  const_reference at(int);
 
 };
 
@@ -207,6 +210,28 @@ void PositiveOperatorCallConstValueParam(const Container<ExpensiveToCopyType> C)
   VarCopyConstructed.constMethod();
 }
 
+void PositiveOperatorValueParam(Container<ExpensiveToCopyType> C) {
+  const auto AutoAssigned = C[42];
+  // CHECK-MESSAGES: [[@LINE-1]]:14: warning: the const qualified variable 'AutoAssigned'
+  // CHECK-FIXES: const auto& AutoAssigned = C[42];
+  AutoAssigned.constMethod();
+
+  const auto AutoCopyConstructed(C[42]);
+  // CHECK-MESSAGES: [[@LINE-1]]:14: warning: the const qualified variable 'AutoCopyConstructed'
+  // CHECK-FIXES: const auto& AutoCopyConstructed(C[42]);
+  AutoCopyConstructed.constMethod();
+
+  const ExpensiveToCopyType VarAssigned = C.at(42);
+  // CHECK-MESSAGES: [[@LINE-1]]:29: warning: the const qualified variable 'VarAssigned'
+  // CHECK-FIXES: const ExpensiveToCopyType& VarAssigned = C.at(42);
+  VarAssigned.constMethod();
+
+  const ExpensiveToCopyType VarCopyConstructed(C.at(42));
+  // CHECK-MESSAGES: [[@LINE-1]]:29: warning: the const qualified variable 'VarCopyConstructed'
+  // CHECK-FIXES: const ExpensiveToCopyType& VarCopyConstructed(C.at(42));
+  VarCopyConstructed.constMethod();
+}
+
 void PositiveOperatorCallConstValueParamAlias(const ExpensiveToCopyContainerAlias C) {
   const auto AutoAssigned = C[42];
   // CHECK-MESSAGES: [[@LINE-1]]:14: warning: the const qualified variable 'AutoAssigned'
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
index ecaf97fa348cc1a4134659b0d39564fe9a2bc6c2..46755270b48ead3d093dbac3e574df6b7c781aa0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
@@ -889,3 +889,9 @@ namespace PR88203 {
     // CHECK-FIXES: {{^    }}if (s.empty()) {}{{$}}
   }
 }
+
+namespace PR94454 {
+  template <char...>
+  int operator""_ci() { return 0; }
+  auto eq = 0_ci == 0;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx20.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx20.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..13aa5c5774b472e1416eabb1cbb2faa8079e829b
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx20.cpp
@@ -0,0 +1,31 @@
+// RUN: %check_clang_tidy -std=c++20 %s readability-implicit-bool-conversion %t
+
+namespace std {
+struct strong_ordering {
+  int n;
+  constexpr operator int() const { return n; }
+  static const strong_ordering equal, greater, less;
+};
+constexpr strong_ordering strong_ordering::equal = {0};
+constexpr strong_ordering strong_ordering::greater = {1};
+constexpr strong_ordering strong_ordering::less = {-1};
+} // namespace std
+
+namespace PR93409 {
+  struct X
+  {
+      auto operator<=>(const X&) const = default;
+      bool m_b;
+  };
+
+  struct Y
+  {
+      auto operator<=>(const Y&) const = default;
+      X m_x;
+  };
+  
+  bool compare(const Y& y1, const Y& y2)
+  {
+     return y1 == y2 || y1 < y2 || y1 > y2;
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
index a6045c079a4823bb0852e64e15a0d8853bd13bc7..4face0bb3fe6805e1407c38cecd3b0e9b031b298 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
@@ -140,3 +140,20 @@ void f(){
     //CHECK-MESSAGES: :[[@LINE+1]]:13: warning: '*' has higher precedence than '+'; add parentheses to explicitly specify the order of operations [readability-math-missing-parentheses]
     int v = FUN5(0 + 1);
 }
+
+namespace PR92516 {
+  void f(int i) {
+    int j, k;
+    for (j = i + 1, k = 0; j < 1; ++j) {}
+  }
+
+  void f2(int i) {
+    int j;
+    for (j = i + 1; j < 1; ++j) {}
+  }
+
+  void f3(int i) {
+    int j;
+    for (j = i + 1, 2; j < 1; ++j) {}
+  }
+}
diff --git a/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp b/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp
index 3d9f51e2e17b0945bec574145a46a3437f9402df..064e04c932de83e212a035be7da8bf7c88a4f493 100644
--- a/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp
@@ -46,6 +46,7 @@ template <int Indirections> void RunTest(StringRef Snippet) {
   StringRef CommonCode = R"(
     struct ConstTag{};
     struct NonConstTag{};
+    struct Tag1{};
 
     struct S {
       void constMethod() const;
@@ -59,6 +60,13 @@ template <int Indirections> void RunTest(StringRef Snippet) {
       void operator[](int);
       void operator[](int) const;
 
+      int& at(int);
+      const int& at(int) const;
+      const int& at(Tag1);
+
+      int& weird_overload();
+      const double& weird_overload() const;
+
       bool operator==(const S&) const;
 
       int int_member;
@@ -161,9 +169,11 @@ TEST(ConstReferenceDeclRefExprsTest, ConstRefVar) {
       useIntConstRef(/*const*/target.int_member);
       useIntPtr(/*const*/target.ptr_member);
       useIntConstPtr(&/*const*/target.int_member);
+      (void)/*const*/target.at(3);
 
       const S& const_target_ref = /*const*/target;
       const S* const_target_ptr = &/*const*/target;
+      (void)/*const*/target.at(3);
     }
 )");
 }
@@ -187,7 +197,7 @@ TEST(ConstReferenceDeclRefExprsTest, ValueVar) {
       /*const*/target.staticMethod();
       target.nonConstMethod();
       /*const*/target(ConstTag{});
-      target[42];
+      /*const*/target[42];
       /*const*/target(ConstTag{});
       target(NonConstTag{});
       useRef(target);
@@ -211,6 +221,14 @@ TEST(ConstReferenceDeclRefExprsTest, ValueVar) {
       const S& const_target_ref = /*const*/target;
       const S* const_target_ptr = &/*const*/target;
       S* target_ptr = &target;
+
+      (void)/*const*/target.at(3);
+      ++target.at(3);
+      const int civ = /*const*/target.at(3);
+      const int& cir = /*const*/target.at(3);
+      int& ir = target.at(3);
+      target.at(Tag1{});
+      target.weird_overload();
     }
 )");
 }
@@ -227,7 +245,7 @@ TEST(ConstReferenceDeclRefExprsTest, RefVar) {
       /*const*/target.staticMethod();
       target.nonConstMethod();
       /*const*/target(ConstTag{});
-      target[42];
+      /*const*/target[42];
       useConstRef((/*const*/target));
       (/*const*/target).constMethod();
       (void)(/*const*/target == /*const*/target);
@@ -249,6 +267,14 @@ TEST(ConstReferenceDeclRefExprsTest, RefVar) {
       const S& const_target_ref = /*const*/target;
       const S* const_target_ptr = &/*const*/target;
       S* target_ptr = &target;
+
+      (void)/*const*/target.at(3);
+      ++target.at(3);
+      const int civ = /*const*/target.at(3);
+      const int& cir = /*const*/target.at(3);
+      int& ir = target.at(3);
+      target.at(Tag1{});
+      target.weird_overload();
     }
 )");
 }
@@ -266,8 +292,8 @@ TEST(ConstReferenceDeclRefExprsTest, PtrVar) {
       /*const*/target->staticMethod();
       target->nonConstMethod();
       (*/*const*/target)(ConstTag{});
-      (*target)[42];
-      target->operator[](42);
+      (*/*const*/target)[42];
+      /*const*/target->operator[](42);
       useConstRef((*/*const*/target));
       (/*const*/target)->constMethod();
       (void)(*/*const*/target == */*const*/target);
@@ -284,7 +310,15 @@ TEST(ConstReferenceDeclRefExprsTest, PtrVar) {
 
       const S& const_target_ref = */*const*/target;
       const S* const_target_ptr = /*const*/target;
-      S* target_ptr = target;  // FIXME: we could chect const usage of `target_ptr`.
+      S* target_ptr = target;  // FIXME: we could chect const usage of `target_ptr`
+
+      (void)/*const*/target->at(3);
+      ++target->at(3);
+      const int civ = /*const*/target->at(3);
+      const int& cir = /*const*/target->at(3);
+      int& ir = target->at(3);
+      target->at(Tag1{});
+      target->weird_overload();
     }
 )");
 }
@@ -319,6 +353,10 @@ TEST(ConstReferenceDeclRefExprsTest, ConstPtrVar) {
 
       const S& const_target_ref = */*const*/target;
       const S* const_target_ptr = /*const*/target;
+
+      (void)/*const*/target->at(3);
+      const int civ = /*const*/target->at(3);
+      const int& cir = /*const*/target->at(3);
     }
 )");
 }
diff --git a/clang/cmake/caches/CrossWinToARMLinux.cmake b/clang/cmake/caches/CrossWinToARMLinux.cmake
index 6826d01f8b2a7381fb9c9c4bc245d2f6c7816bee..e4d0a0c2d14cb9a1d3589fd440969761cd34c0a3 100644
--- a/clang/cmake/caches/CrossWinToARMLinux.cmake
+++ b/clang/cmake/caches/CrossWinToARMLinux.cmake
@@ -6,21 +6,23 @@
 # on Windows platform.
 #
 # NOTE: the build requires a development ARM Linux root filesystem to use
-# proper target platform depended library and header files:
-#  - create <full-path-to-clang-configs> directory and put the clang configuration
-#    file named <TOOLCHAIN_TARGET_TRIPLE>.cfg into it.
-#  - add the `--sysroot=<path-to-develop-arm-linux-root-fs>` argument into
-#    this configuration file.
-#  - add other necessary target depended clang arguments there, 
-#    such as '-mcpu=cortex-a78' & etc.
+# proper target platform depended library and header files.
+#
+# The build generates a proper clang configuration file with stored
+# --sysroot argument for specified target triple. Also it is possible
+# to specify configuration path via CMake arguments, such as
+#   -DCLANG_CONFIG_FILE_USER_DIR=<full-path-to-clang-configs>
+# and/or
+#   -DCLANG_CONFIG_FILE_SYSTEM_DIR=<full-path-to-clang-configs>
 #
 # See more details here: https://clang.llvm.org/docs/UsersManual.html#configuration-files
 #
 # Configure:
 #  cmake -G Ninja ^
 #       -DTOOLCHAIN_TARGET_TRIPLE=aarch64-unknown-linux-gnu ^
+#       -DTOOLCHAIN_TARGET_SYSROOTFS=<path-to-develop-arm-linux-root-fs> ^
+#       -DTOOLCHAIN_SHARED_LIBS=OFF ^ 
 #       -DCMAKE_INSTALL_PREFIX=../install ^
-#       -DCLANG_CONFIG_FILE_USER_DIR=<full-path-to-clang-configs> ^
 #       -DCMAKE_CXX_FLAGS="-D__OPTIMIZE__" ^
 #       -DREMOTE_TEST_HOST="<hostname>" ^
 #       -DREMOTE_TEST_USER="<ssh_user_name>" ^
@@ -81,6 +83,20 @@ endif()
 
 message(STATUS "Toolchain target triple: ${TOOLCHAIN_TARGET_TRIPLE}")
 
+if (DEFINED TOOLCHAIN_TARGET_SYSROOTFS)
+  message(STATUS "Toolchain target sysroot: ${TOOLCHAIN_TARGET_SYSROOTFS}")
+  # Store the --sysroot argument for the compiler-rt test flags.
+  set(sysroot_flags --sysroot='${TOOLCHAIN_TARGET_SYSROOTFS}')
+  # Generate the clang configuration file for the specified target triple
+  # and store --sysroot in this file.
+  file(WRITE "${CMAKE_BINARY_DIR}/bin/${TOOLCHAIN_TARGET_TRIPLE}.cfg" ${sysroot_flags})
+endif()
+
+# Build the shared libraries for libc++/libc++abi/libunwind.
+if (NOT DEFINED TOOLCHAIN_SHARED_LIBS)
+  set(TOOLCHAIN_SHARED_LIBS OFF)
+endif()
+ 
 if (NOT DEFINED LLVM_TARGETS_TO_BUILD)
   if ("${TOOLCHAIN_TARGET_TRIPLE}" MATCHES "^(armv|arm32)+")
     set(LLVM_TARGETS_TO_BUILD "ARM" CACHE STRING "")
@@ -183,20 +199,21 @@ set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_CAN_EXECUTE_TESTS
 
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_USE_BUILTINS_LIBRARY          ON CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_CXX_LIBRARY                   libcxx CACHE STRING "")
-# Tell Clang to seach C++ headers alongside with the just-built binaries for the C++ compiler-rt tests.
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_TEST_COMPILER_CFLAGS          "--stdlib=libc++" CACHE STRING "")
-
+# The compiler-rt tests disable the clang configuration files during the execution by setting CLANG_NO_DEFAULT_CONFIG=1
+# and drops out the --sysroot from there. Provide it explicity via the test flags here if target sysroot has been specified.
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_COMPILER_RT_TEST_COMPILER_CFLAGS          "--stdlib=libc++ ${sysroot_flags}" CACHE STRING "")
+  
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_USE_COMPILER_RT                 ON CACHE BOOL "")
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_ENABLE_SHARED                   OFF CACHE BOOL "")
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_ENABLE_SHARED                   ${TOOLCHAIN_SHARED_LIBS} CACHE BOOL "")
 
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_USE_LLVM_UNWINDER               ON CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_STATIC_UNWINDER          ON CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_USE_COMPILER_RT                 ON CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS   OFF CACHE BOOL "")
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_SHARED                   OFF CACHE BOOL "")
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_SHARED                   ${TOOLCHAIN_SHARED_LIBS} CACHE BOOL "")
 
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_USE_COMPILER_RT                    ON CACHE BOOL "")
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_SHARED                      OFF CACHE BOOL "")
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_SHARED                      ${TOOLCHAIN_SHARED_LIBS} CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ABI_VERSION                        ${LIBCXX_ABI_VERSION} CACHE STRING "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_CXX_ABI                            "libcxxabi" CACHE STRING "")    #!!!
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS      ON CACHE BOOL "")
diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index c0bfcbdfc1c2ae6cb32edfcb26e376baa1dd2641..1dfb1bc535bf149c4288b8f527287e1c2d296822 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -91,4 +91,6 @@ endif()
 # Final Stage Config (stage2)
 set_final_stage_var(LLVM_ENABLE_RUNTIMES "${LLVM_RELEASE_ENABLE_RUNTIMES}" STRING)
 set_final_stage_var(LLVM_ENABLE_PROJECTS "${LLVM_RELEASE_ENABLE_PROJECTS}" STRING)
+set_final_stage_var(CPACK_GENERATOR "TXZ" STRING)
+set_final_stage_var(CPACK_ARCHIVE_THREADS "0" STRING)
 
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 46f99d0bbdd066932de0adbb7cb83ff5322ae36e..a49e4122ffc100801a2d40c34a13e3938e2ee79e 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -4016,6 +4016,30 @@ Note that the `size` argument must be a compile time constant.
 
 Note that this intrinsic cannot yet be called in a ``constexpr`` context.
 
+``__is_bitwise_cloneable``
+--------------------------
+
+A type trait is used to check whether a type can be safely copied by memcpy.
+
+**Syntax**:
+
+.. code-block:: c++
+
+  bool __is_bitwise_cloneable(Type)
+
+**Description**:
+
+Objects of bitwise cloneable types can be bitwise copied by memcpy/memmove. The
+Clang compiler warrants that this behavior is well defined, and won't be
+broken by compiler optimizations and sanitizers.
+
+For implicit-lifetime types, the lifetime of the new object is implicitly
+started after the copy. For other types (e.g., classes with virtual methods),
+the lifetime isn't started, and using the object results in undefined behavior
+according to the C++ Standard.
+
+This builtin can be used in constant expressions.
+
 Atomic Min/Max builtins with memory ordering
 --------------------------------------------
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 69ac08133c9f0d1a6e8e8128d1ef7ac4924226f5..cf1ba02cbc4b2f59534acf4c43a7153f65b32134 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -340,6 +340,9 @@ Non-comprehensive list of changes in this release
   ``-Winvalid-constexpr`` is not enabled for the function definition, which
   should result in mild compile-time performance improvements.
 
+- Added ``__is_bitwise_cloneable`` which is used to check whether a type
+  can be safely copied by memcpy/memmove.
+
 New Compiler Flags
 ------------------
 - ``-fsanitize=implicit-bitfield-conversion`` checks implicit truncation and
@@ -904,6 +907,7 @@ Arm and AArch64 Support
     * Arm Cortex-A520AE (cortex-a520ae).
     * Arm Cortex-A720AE (cortex-a720ae).
     * Arm Cortex-R82AE (cortex-r82ae).
+    * Arm Cortex-R52+ (cortex-r52plus).
     * Arm Neoverse-N3 (neoverse-n3).
     * Arm Neoverse-V3 (neoverse-v3).
     * Arm Neoverse-V3AE (neoverse-v3ae).
diff --git a/clang/examples/PrintFunctionNames/PrintFunctionNames.cpp b/clang/examples/PrintFunctionNames/PrintFunctionNames.cpp
index 6509a6440e12d5d7e67943a2c86396ea7a4cb1e7..b2b785b87c25cf0975eb84350f6d7cd5b32c45cb 100644
--- a/clang/examples/PrintFunctionNames/PrintFunctionNames.cpp
+++ b/clang/examples/PrintFunctionNames/PrintFunctionNames.cpp
@@ -72,7 +72,7 @@ public:
           *sema.LateParsedTemplateMap.find(FD)->second;
       sema.LateTemplateParser(sema.OpaqueParser, LPT);
       llvm::errs() << "late-parsed-decl: \"" << FD->getNameAsString() << "\"\n";
-    }   
+    }
   }
 };
 
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index a1d1d1c51cd417d18d4253eedb510b1e05f34f3e..8bce4812f0d482e7897ddba1a799f27daa79be52 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -3203,9 +3203,6 @@ public:
   /// valid feature names.
   ParsedTargetAttr filterFunctionTargetAttrs(const TargetAttr *TD) const;
 
-  std::vector<std::string>
-  filterFunctionTargetVersionAttrs(const TargetVersionAttr *TV) const;
-
   void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                              const FunctionDecl *) const;
   void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
diff --git a/clang/include/clang/AST/ASTUnresolvedSet.h b/clang/include/clang/AST/ASTUnresolvedSet.h
index 398ffb188c95b04abe266b104f9fe722926ede2f..dcce3bc63df2521f5f79cc9ea70b0b9d9e3918a1 100644
--- a/clang/include/clang/AST/ASTUnresolvedSet.h
+++ b/clang/include/clang/AST/ASTUnresolvedSet.h
@@ -16,6 +16,7 @@
 
 #include "clang/AST/ASTVector.h"
 #include "clang/AST/DeclAccessPair.h"
+#include "clang/AST/DeclID.h"
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/Basic/Specifiers.h"
 #include <cassert>
@@ -56,6 +57,10 @@ public:
     Decls.push_back(DeclAccessPair::make(D, AS), C);
   }
 
+  void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) {
+    Decls.push_back(DeclAccessPair::makeLazy(ID.get(), AS), C);
+  }
+
   /// Replaces the given declaration with the new one, once.
   ///
   /// \return true if the set changed
@@ -109,10 +114,10 @@ public:
 
   void reserve(ASTContext &C, unsigned N) { Impl.reserve(C, N); }
 
-  void addLazyDecl(ASTContext &C, uintptr_t ID, AccessSpecifier AS) {
+  void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) {
     assert(Impl.empty() || Impl.Decls.isLazy());
     Impl.Decls.setLazy(true);
-    Impl.addDecl(C, reinterpret_cast<NamedDecl *>(ID << 2), AS);
+    Impl.addLazyDecl(C, ID, AS);
   }
 };
 
diff --git a/clang/include/clang/AST/DeclAccessPair.h b/clang/include/clang/AST/DeclAccessPair.h
index 805342c2910a7177a9f1403d8472a837c35b1b8f..4becfde963057247f8a5d56fe76a6fc548aa8a33 100644
--- a/clang/include/clang/AST/DeclAccessPair.h
+++ b/clang/include/clang/AST/DeclAccessPair.h
@@ -19,6 +19,7 @@
 
 #include "clang/Basic/Specifiers.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Endian.h"
 
 namespace clang {
 
@@ -27,9 +28,17 @@ class NamedDecl;
 /// A POD class for pairing a NamedDecl* with an access specifier.
 /// Can be put into unions.
 class DeclAccessPair {
-  uintptr_t Ptr; // we'd use llvm::PointerUnion, but it isn't trivial
+  /// Use the lower 2 bit to store AccessSpecifier. Use the higher
+  /// 61 bit to store the pointer to a NamedDecl or the DeclID to
+  /// a NamedDecl. If the 3rd bit is set, storing the DeclID, otherwise
+  /// storing the pointer.
+  llvm::support::detail::packed_endian_specific_integral<
+      uint64_t, llvm::endianness::native, alignof(void *)>
+      Ptr;
 
-  enum { Mask = 0x3 };
+  enum { ASMask = 0x3, Mask = 0x7 };
+
+  bool isDeclID() const { return (Ptr >> 2) & 0x1; }
 
 public:
   static DeclAccessPair make(NamedDecl *D, AccessSpecifier AS) {
@@ -38,12 +47,22 @@ public:
     return p;
   }
 
+  static DeclAccessPair makeLazy(uint64_t ID, AccessSpecifier AS) {
+    DeclAccessPair p;
+    p.Ptr = (ID << 3) | (0x1 << 2) | uint64_t(AS);
+    return p;
+  }
+
+  uint64_t getDeclID() const {
+    assert(isDeclID());
+    return (~Mask & Ptr) >> 3;
+  }
+
   NamedDecl *getDecl() const {
+    assert(!isDeclID());
     return reinterpret_cast<NamedDecl*>(~Mask & Ptr);
   }
-  AccessSpecifier getAccess() const {
-    return AccessSpecifier(Mask & Ptr);
-  }
+  AccessSpecifier getAccess() const { return AccessSpecifier(ASMask & Ptr); }
 
   void setDecl(NamedDecl *D) {
     set(D, getAccess());
@@ -52,12 +71,18 @@ public:
     set(getDecl(), AS);
   }
   void set(NamedDecl *D, AccessSpecifier AS) {
-    Ptr = uintptr_t(AS) | reinterpret_cast<uintptr_t>(D);
+    Ptr = uint64_t(AS) | reinterpret_cast<uint64_t>(D);
   }
 
   operator NamedDecl*() const { return getDecl(); }
   NamedDecl *operator->() const { return getDecl(); }
 };
+
+// Make sure DeclAccessPair is pointer-aligned types.
+static_assert(alignof(DeclAccessPair) == alignof(void *));
+// Make sure DeclAccessPair is still POD.
+static_assert(std::is_standard_layout_v<DeclAccessPair> &&
+              std::is_trivial_v<DeclAccessPair>);
 }
 
 #endif
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 600ce73c7f01995adcfaa0a1bd05997289e02aef..5f19af1891b74284358dd8195cb3572cee53ba21 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -708,10 +708,7 @@ public:
 
   /// Set the owning module ID.  This may only be called for
   /// deserialized Decls.
-  void setOwningModuleID(unsigned ID) {
-    assert(isFromASTFile() && "Only works on a deserialized declaration");
-    *((unsigned*)this - 2) = ID;
-  }
+  void setOwningModuleID(unsigned ID);
 
 public:
   /// Determine the availability of the given declaration.
@@ -784,19 +781,11 @@ public:
 
   /// Retrieve the global declaration ID associated with this
   /// declaration, which specifies where this Decl was loaded from.
-  GlobalDeclID getGlobalID() const {
-    if (isFromASTFile())
-      return (*((const GlobalDeclID *)this - 1));
-    return GlobalDeclID();
-  }
+  GlobalDeclID getGlobalID() const;
 
   /// Retrieve the global ID of the module that owns this particular
   /// declaration.
-  unsigned getOwningModuleID() const {
-    if (isFromASTFile())
-      return *((const unsigned*)this - 2);
-    return 0;
-  }
+  unsigned getOwningModuleID() const;
 
 private:
   Module *getOwningModuleSlow() const;
diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h
index 614ba06b63860cdd13f7c1f2761c40231ca39275..32d2ed41a374a096636a204c4bc06e02f64ed6c7 100644
--- a/clang/include/clang/AST/DeclID.h
+++ b/clang/include/clang/AST/DeclID.h
@@ -19,6 +19,8 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/iterator.h"
 
+#include <climits>
+
 namespace clang {
 
 /// Predefined declaration IDs.
@@ -107,12 +109,16 @@ public:
   ///
   /// DeclID should only be used directly in serialization. All other users
   /// should use LocalDeclID or GlobalDeclID.
-  using DeclID = uint32_t;
+  using DeclID = uint64_t;
 
 protected:
   DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {}
   explicit DeclIDBase(DeclID ID) : ID(ID) {}
 
+  explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) {
+    ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32);
+  }
+
 public:
   DeclID get() const { return ID; }
 
@@ -124,6 +130,10 @@ public:
 
   bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; }
 
+  unsigned getModuleFileIndex() const { return ID >> 32; }
+
+  unsigned getLocalDeclIndex() const;
+
   friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) {
     return LHS.ID == RHS.ID;
   }
@@ -156,6 +166,9 @@ public:
   LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {}
   explicit LocalDeclID(DeclID ID) : Base(ID) {}
 
+  explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex)
+      : Base(LocalID, ModuleFileIndex) {}
+
   LocalDeclID &operator++() {
     ++ID;
     return *this;
@@ -175,6 +188,9 @@ public:
   GlobalDeclID() : Base() {}
   explicit GlobalDeclID(DeclID ID) : Base(ID) {}
 
+  explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex)
+      : Base(LocalID, ModuleFileIndex) {}
+
   // For DeclIDIterator<GlobalDeclID> to be able to convert a GlobalDeclID
   // to a LocalDeclID.
   explicit operator LocalDeclID() const { return LocalDeclID(this->ID); }
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index a4c82cdad87bda0caad72282bb285e5b4a8d6003..ea1ffbc7fd08b463ffa9aff0b32fa9ee44e6dc10 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -867,7 +867,7 @@ public:
   case OpenACCClauseKind::CLAUSE_NAME:                                         \
     Visit##CLAUSE_NAME##Clause(*cast<OpenACC##CLAUSE_NAME##Clause>(C));        \
     return;
-#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME)                                  \
+#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED)                      \
   case OpenACCClauseKind::ALIAS_NAME:                                          \
     Visit##CLAUSE_NAME##Clause(*cast<OpenACC##CLAUSE_NAME##Clause>(C));        \
     return;
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 263b632df23ce42d00e9d04c6b7acbdf0b3d6df6..9eb3f6c09e3d3b188d3e316f320283e61a8498a9 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1120,6 +1120,20 @@ public:
   /// Return true if this is a trivially copyable type (C++0x [basic.types]p9)
   bool isTriviallyCopyableType(const ASTContext &Context) const;
 
+  /// Return true if the type is safe to bitwise copy using memcpy/memmove.
+  ///
+  /// This is an extension in clang: bitwise cloneable types act as trivially
+  /// copyable types, meaning their underlying bytes can be safely copied by
+  /// memcpy or memmove. After the copy, the destination object has the same
+  /// object representation.
+  ///
+  /// However, there are cases where it is not safe to copy:
+  ///  - When sanitizers, such as AddressSanitizer, add padding with poison,
+  ///    which can cause issues if those poisoned padding bits are accessed.
+  ///  - Types with Objective-C lifetimes, where specific runtime
+  ///    semantics may not be preserved during a bitwise copy.
+  bool isBitwiseCloneableType(const ASTContext &Context) const;
+
   /// Return true if this is a trivially copyable type
   bool isTriviallyCopyConstructibleType(const ASTContext &Context) const;
 
diff --git a/clang/include/clang/AST/UnresolvedSet.h b/clang/include/clang/AST/UnresolvedSet.h
index ee31be969b6e3585dcfbe9cc290003ae744f356a..1369725ab4e96adfb17d6b8b4cbba3b4c47e3786 100644
--- a/clang/include/clang/AST/UnresolvedSet.h
+++ b/clang/include/clang/AST/UnresolvedSet.h
@@ -47,6 +47,7 @@ public:
   // temporaries with defaulted ctors are not zero initialized.
   UnresolvedSetIterator() : iterator_adaptor_base(nullptr) {}
 
+  uint64_t getDeclID() const { return I->getDeclID(); }
   NamedDecl *getDecl() const { return I->getDecl(); }
   void setDecl(NamedDecl *ND) const { return I->setDecl(ND); }
   AccessSpecifier getAccess() const { return I->getAccess(); }
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 17d9a710d948b24d9e8781b1611bdf8f920b5cbc..b70b0c8b836a547547979f90a9af684089256503 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4470,37 +4470,20 @@ def HLSLShader : InheritableAttr {
   let Subjects = SubjectList<[HLSLEntry]>;
   let LangOpts = [HLSL];
   let Args = [
-    EnumArgument<"Type", "ShaderType", /*is_string=*/true,
+    EnumArgument<"Type", "llvm::Triple::EnvironmentType", /*is_string=*/true,
                  ["pixel", "vertex", "geometry", "hull", "domain", "compute",
                   "raygeneration", "intersection", "anyhit", "closesthit",
                   "miss", "callable", "mesh", "amplification"],
                  ["Pixel", "Vertex", "Geometry", "Hull", "Domain", "Compute",
                   "RayGeneration", "Intersection", "AnyHit", "ClosestHit",
-                  "Miss", "Callable", "Mesh", "Amplification"]>
+                  "Miss", "Callable", "Mesh", "Amplification"],
+                  /*opt=*/0, /*fake=*/0, /*isExternalType=*/1>
   ];
   let Documentation = [HLSLSV_ShaderTypeAttrDocs];
   let AdditionalMembers =
 [{
-  static const unsigned ShaderTypeMaxValue = (unsigned)HLSLShaderAttr::Amplification;
-
-  static llvm::Triple::EnvironmentType getTypeAsEnvironment(HLSLShaderAttr::ShaderType ShaderType) {
-    switch (ShaderType) {
-      case HLSLShaderAttr::Pixel:         return llvm::Triple::Pixel;
-      case HLSLShaderAttr::Vertex:        return llvm::Triple::Vertex;
-      case HLSLShaderAttr::Geometry:      return llvm::Triple::Geometry;
-      case HLSLShaderAttr::Hull:          return llvm::Triple::Hull;
-      case HLSLShaderAttr::Domain:        return llvm::Triple::Domain;
-      case HLSLShaderAttr::Compute:       return llvm::Triple::Compute;
-      case HLSLShaderAttr::RayGeneration: return llvm::Triple::RayGeneration;
-      case HLSLShaderAttr::Intersection:  return llvm::Triple::Intersection;
-      case HLSLShaderAttr::AnyHit:        return llvm::Triple::AnyHit;
-      case HLSLShaderAttr::ClosestHit:    return llvm::Triple::ClosestHit;
-      case HLSLShaderAttr::Miss:          return llvm::Triple::Miss;
-      case HLSLShaderAttr::Callable:      return llvm::Triple::Callable;
-      case HLSLShaderAttr::Mesh:          return llvm::Triple::Mesh;
-      case HLSLShaderAttr::Amplification: return llvm::Triple::Amplification;
-    }
-    llvm_unreachable("unknown enumeration value");
+  static bool isValidShaderType(llvm::Triple::EnvironmentType ShaderType) {
+    return ShaderType >= llvm::Triple::Pixel && ShaderType <= llvm::Triple::Amplification;
   }
 }];
 }
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 433c7795325f0c75c7418be4b9df8273bed172f3..9e6800ea814a0aff9c95c24cb304ea3e185bcf1e 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -240,7 +240,7 @@ TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_v2bf16, "V2sV2s*0V2s", "t", "at
 TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fadd_v2bf16, "V2sV2s*1V2s", "t", "atomic-global-pk-add-bf16-inst")
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2bf16, "V2sV2s*3V2s", "t", "atomic-ds-pk-add-16-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2f16, "V2hV2h*3V2h", "t", "atomic-ds-pk-add-16-insts")
-TARGET_BUILTIN(__builtin_amdgcn_global_load_lds, "vv*1v*3UiiUi", "t", "gfx940-insts")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_lds, "vv*1v*3IUiIiIUi", "t", "gfx940-insts")
 
 //===----------------------------------------------------------------------===//
 // Deep learning builtins.
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 07b0ca1691a679a2148a72734e2a658bb5286d58..7ffc40a00504fb9fb9058dd73880cd2608a8d6a3 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -61,7 +61,7 @@ CODEGENOPT(SeparateNamedSections, 1, 0) ///< Set for -fseparate-named-sections.
 CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX.
 CODEGENOPT(XCOFFReadOnlyPointers, 1, 0) ///< Set for -mxcoff-roptr.
 CODEGENOPT(AllTocData, 1, 0) ///< AIX -mtocdata
-ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,none
+ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,reserved,none
 
 CODEGENOPT(ClearASTBeforeBackend , 1, 0) ///< Free the AST before running backend code generation. Only works with -disable-free.
 CODEGENOPT(DisableFree       , 1, 0) ///< Don't free memory.
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 9469a424045bb0472a6cd725ebf55bfa1e0e08d8..00523a84d3895f14279c1180a25de5800b2ae8bd 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -127,15 +127,18 @@ public:
   std::string BinutilsVersion;
 
   enum class FramePointerKind {
-    None,        // Omit all frame pointers.
-    NonLeaf,     // Keep non-leaf frame pointers.
-    All,         // Keep all frame pointers.
+    None,     // Omit all frame pointers.
+    Reserved, // Maintain valid frame pointer chain.
+    NonLeaf,  // Keep non-leaf frame pointers.
+    All,      // Keep all frame pointers.
   };
 
   static StringRef getFramePointerKindName(FramePointerKind Kind) {
     switch (Kind) {
     case FramePointerKind::None:
       return "none";
+    case FramePointerKind::Reserved:
+      return "reserved";
     case FramePointerKind::NonLeaf:
       return "non-leaf";
     case FramePointerKind::All:
diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index d15171d959c45abc4c92125743b91804d99f448b..0d5e38e825aa78e44f0560d1fb8c3f0a33c70ced 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -124,6 +124,7 @@ enum class CudaArch {
   GFX1103,
   GFX1150,
   GFX1151,
+  GFX1152,
   GFX12_GENERIC,
   GFX1200,
   GFX1201,
diff --git a/clang/include/clang/Basic/DiagnosticOptions.h b/clang/include/clang/Basic/DiagnosticOptions.h
index 099982c3bdd5a00a5b4b2cd7787d01f51b11ed4e..30141c2b8f447548fbd6f74217d59c3d74eb1464 100644
--- a/clang/include/clang/Basic/DiagnosticOptions.h
+++ b/clang/include/clang/Basic/DiagnosticOptions.h
@@ -124,7 +124,7 @@ public:
   /// default).
   std::vector<std::string> VerifyPrefixes;
 
-  /// The list of -Wsystem-header-in-module=... options used to override
+  /// The list of -Wsystem-headers-in-module=... options used to override
   /// whether -Wsystem-headers is enabled on a per-module basis.
   std::vector<std::string> SystemHeaderWarningsModules;
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 87745140cb0ebd187331eb463f611f463ae07663..de3aa4b4028eb1c820596c99abebf2b8f64fe9cc 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -607,6 +607,8 @@ def note_using_decl_class_member_workaround : Note<
   "a const variable|a constexpr variable}0 instead">;
 def err_using_decl_can_not_refer_to_namespace : Error<
   "using declaration cannot refer to a namespace">;
+def note_namespace_using_decl : Note<
+  "did you mean 'using namespace'?">;
 def warn_cxx17_compat_using_decl_scoped_enumerator: Warning<
   "using declaration naming a scoped enumerator is incompatible with "
   "C++ standards before C++20">, InGroup<CXXPre20Compat>, DefaultIgnore;
@@ -6088,9 +6090,9 @@ def err_redefinition_different_concept : Error<
   "redefinition of concept %0 with different template parameters or requirements">;
 def err_tag_reference_non_tag : Error<
   "%select{non-struct type|non-class type|non-union type|non-enum "
-  "type|typedef|type alias|template|type alias template|template "
-  "template argument}1 %0 cannot be referenced with a "
-  "%select{struct|interface|union|class|enum}2 specifier">;
+  "type|typedef|type alias|template|alias template|template "
+  "template argument}1 %0 cannot be referenced with the '"
+  "%select{struct|interface|union|class|enum}2' specifier">;
 def err_tag_reference_conflict : Error<
   "implicit declaration introduced by elaborated type conflicts with a "
   "%select{non-struct type|non-class type|non-union type|non-enum "
@@ -10082,6 +10084,12 @@ def warn_new_dangling_initializer_list : Warning<
   "the allocated initializer list}0 "
   "will be destroyed at the end of the full-expression">,
   InGroup<DanglingInitializerList>;
+def warn_unsupported_lifetime_extension : Warning<
+  "lifetime extension of "
+  "%select{temporary|backing array of initializer list}0 created "
+  "by aggregate initialization using a default member initializer "
+  "is not yet supported; lifetime of %select{temporary|backing array}0 "
+  "will end at the end of the full-expression">, InGroup<Dangling>;
 
 // For non-floating point, expressions of the form x == x or x != x
 // should result in a warning, since these always evaluate to a constant.
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index 53f4cd103d142ce6cf915e200e4ad9e9064ccd9e..85f4859925f0bc6982a1854ec3567ae94482e915 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -15,31 +15,31 @@
 //
 // VISIT_CLAUSE(CLAUSE_NAME)
 //
-// CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME)
+// CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED)
 
 #ifndef CLAUSE_ALIAS
-#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME)
+#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, false)
 #endif
 
 VISIT_CLAUSE(Auto)
 VISIT_CLAUSE(Async)
 VISIT_CLAUSE(Attach)
 VISIT_CLAUSE(Copy)
-CLAUSE_ALIAS(PCopy, Copy)
-CLAUSE_ALIAS(PresentOrCopy, Copy)
+CLAUSE_ALIAS(PCopy, Copy, true)
+CLAUSE_ALIAS(PresentOrCopy, Copy, true)
 VISIT_CLAUSE(CopyIn)
-CLAUSE_ALIAS(PCopyIn, CopyIn)
-CLAUSE_ALIAS(PresentOrCopyIn, CopyIn)
+CLAUSE_ALIAS(PCopyIn, CopyIn, true)
+CLAUSE_ALIAS(PresentOrCopyIn, CopyIn, true)
 VISIT_CLAUSE(CopyOut)
-CLAUSE_ALIAS(PCopyOut, CopyOut)
-CLAUSE_ALIAS(PresentOrCopyOut, CopyOut)
+CLAUSE_ALIAS(PCopyOut, CopyOut, true)
+CLAUSE_ALIAS(PresentOrCopyOut, CopyOut, true)
 VISIT_CLAUSE(Create)
-CLAUSE_ALIAS(PCreate, Create)
-CLAUSE_ALIAS(PresentOrCreate, Create)
+CLAUSE_ALIAS(PCreate, Create, true)
+CLAUSE_ALIAS(PresentOrCreate, Create, true)
 VISIT_CLAUSE(Default)
 VISIT_CLAUSE(DevicePtr)
 VISIT_CLAUSE(DeviceType)
-CLAUSE_ALIAS(DType, DeviceType)
+CLAUSE_ALIAS(DType, DeviceType, false)
 VISIT_CLAUSE(FirstPrivate)
 VISIT_CLAUSE(If)
 VISIT_CLAUSE(Independent)
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index b5a0e9df9f7aeea1301250f5308a11d8799e0708..9c4b17465e18a178ba56cb472d651da3eed5888b 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -542,6 +542,8 @@ TYPE_TRAIT_2(__reference_converts_from_temporary, ReferenceConvertsFromTemporary
 // is not exposed to users.
 TYPE_TRAIT_2(/*EmptySpellingName*/, IsDeducible, KEYCXX)
 
+TYPE_TRAIT_1(__is_bitwise_cloneable, IsBitwiseCloneable, KEYALL)
+
 // Embarcadero Expression Traits
 EXPRESSION_TRAIT(__is_lvalue_expr, IsLValueExpr, KEYCXX)
 EXPRESSION_TRAIT(__is_rvalue_expr, IsRValueExpr, KEYCXX)
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index cca4367751b92b0812c6e16bc084d64aceea641e..a0820e2093bc2071a6f1f9207f2927a094702aee 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -2637,7 +2637,8 @@ let UnMaskedPolicyScheme = HasPassthruOperand in {
     defm vbrev   : RVVOutBuiltinSetZvbb;
     defm vclz    : RVVOutBuiltinSetZvbb;
     defm vctz    : RVVOutBuiltinSetZvbb;
-    defm vcpopv  : RVVOutBuiltinSetZvbb;
+    let IRName = "vcpopv", MaskedIRName = "vcpopv_mask" in
+    defm vcpop   : RVVOutBuiltinSetZvbb;
     let OverloadedName = "vwsll" in
     defm vwsll   : RVVSignedWidenBinBuiltinSetVwsll;
   }
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index f04c220d6e1dbe6a8fa6202dd61d2edd7ccbebfa..d44faa55c456f24cb449f211f12f8263c0e9efec 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -7724,8 +7724,8 @@ def pic_is_pie : Flag<["-"], "pic-is-pie">,
   MarshallingInfoFlag<LangOpts<"PIE">>;
 
 def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">,
-  HelpText<"Specify which frame pointers to retain.">, Values<"all,non-leaf,none">,
-  NormalizedValuesScope<"CodeGenOptions::FramePointerKind">, NormalizedValues<["All", "NonLeaf", "None"]>,
+  HelpText<"Specify which frame pointers to retain.">, Values<"all,non-leaf,reserved,none">,
+  NormalizedValuesScope<"CodeGenOptions::FramePointerKind">, NormalizedValues<["All", "NonLeaf", "Reserved", "None"]>,
   MarshallingInfoEnum<CodeGenOpts<"FramePointer">, "None">;
 
 
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index a4f9cad98aa8b1e40d609d001d483e29f2b00077..9789cfacafd780d45b8e04b26fa56b05a83eb5d4 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -205,7 +205,8 @@ protected:
 
   /// Executes the given \p Executable and returns the stdout.
   llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
-  executeToolChainProgram(StringRef Executable) const;
+  executeToolChainProgram(StringRef Executable,
+                          unsigned SecondsToWait = 0) const;
 
   void setTripleEnvironment(llvm::Triple::EnvironmentType Env);
 
diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h
index 2f8354dec939f7cd0ae66e4210dd683b01cd567e..0e115906fbfe51924d8028c7cc024de86a507e6d 100644
--- a/clang/include/clang/Lex/DependencyDirectivesScanner.h
+++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h
@@ -17,7 +17,6 @@
 #ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
 #define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
 
-#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
 
@@ -118,7 +117,7 @@ struct Directive {
 bool scanSourceForDependencyDirectives(
     StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
     SmallVectorImpl<dependency_directives_scan::Directive> &Directives,
-    const LangOptions &LangOpts, DiagnosticsEngine *Diags = nullptr,
+    DiagnosticsEngine *Diags = nullptr,
     SourceLocation InputSourceLoc = SourceLocation());
 
 /// Print the previously scanned dependency directives as minimized source text.
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index 2072cd8d1c3ef8671e4e4a810b9d3ba36a10b8c3..f443e327eaf32985e20393cb53cd18538e6d3525 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -212,7 +212,7 @@ private:
     struct C Capture;
   };
 
-  InitializedEntity() {};
+  InitializedEntity() {}
 
   /// Create the initialization entity for a variable.
   InitializedEntity(VarDecl *Var, EntityKind EK = EK_Variable)
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index e145f5e7f43f85da1b65193126f92f433697bb02..0e41a72e444ef45f9a62470b1ce317f2b5aaf717 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -39,7 +39,7 @@ public:
                                           const AttributeCommonInfo &AL, int X,
                                           int Y, int Z);
   HLSLShaderAttr *mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
-                                  HLSLShaderAttr::ShaderType ShaderType);
+                                  llvm::Triple::EnvironmentType ShaderType);
   HLSLParamModifierAttr *
   mergeParamModifierAttr(Decl *D, const AttributeCommonInfo &AL,
                          HLSLParamModifierAttr::Spelling Spelling);
@@ -48,8 +48,8 @@ public:
   void CheckSemanticAnnotation(FunctionDecl *EntryPoint, const Decl *Param,
                                const HLSLAnnotationAttr *AnnotationAttr);
   void DiagnoseAttrStageMismatch(
-      const Attr *A, HLSLShaderAttr::ShaderType Stage,
-      std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages);
+      const Attr *A, llvm::Triple::EnvironmentType Stage,
+      std::initializer_list<llvm::Triple::EnvironmentType> AllowedStages);
   void DiagnoseAvailabilityViolations(TranslationUnitDecl *TU);
 
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index f59ff6af4c764ceb0144bbad64ea4e916fb9753d..52a6c5e10f802592d14a17b65bd5954546b83107 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -255,6 +255,12 @@ public:
   }
 };
 
+// The unaligned decl ID used in the Blobs of bistreams.
+using unaligned_decl_id_t =
+    llvm::support::detail::packed_endian_specific_integral<
+        serialization::DeclID, llvm::endianness::native,
+        llvm::support::unaligned>;
+
 /// The number of predefined preprocessed entity IDs.
 const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1;
 
@@ -1980,33 +1986,44 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral };
 
 /// Describes the categories of an Objective-C class.
 struct ObjCCategoriesInfo {
-  // The ID of the definition
-  LocalDeclID DefinitionID;
+  // The ID of the definition. Use unaligned_decl_id_t to keep
+  // ObjCCategoriesInfo 32-bit aligned.
+  unaligned_decl_id_t DefinitionID;
 
   // Offset into the array of category lists.
   unsigned Offset;
 
+  ObjCCategoriesInfo() = default;
+  ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset)
+      : DefinitionID(ID.get()), Offset(Offset) {}
+
+  LocalDeclID getDefinitionID() const { return LocalDeclID(DefinitionID); }
+
   friend bool operator<(const ObjCCategoriesInfo &X,
                         const ObjCCategoriesInfo &Y) {
-    return X.DefinitionID < Y.DefinitionID;
+    return X.getDefinitionID() < Y.getDefinitionID();
   }
 
   friend bool operator>(const ObjCCategoriesInfo &X,
                         const ObjCCategoriesInfo &Y) {
-    return X.DefinitionID > Y.DefinitionID;
+    return X.getDefinitionID() > Y.getDefinitionID();
   }
 
   friend bool operator<=(const ObjCCategoriesInfo &X,
                          const ObjCCategoriesInfo &Y) {
-    return X.DefinitionID <= Y.DefinitionID;
+    return X.getDefinitionID() <= Y.getDefinitionID();
   }
 
   friend bool operator>=(const ObjCCategoriesInfo &X,
                          const ObjCCategoriesInfo &Y) {
-    return X.DefinitionID >= Y.DefinitionID;
+    return X.getDefinitionID() >= Y.getDefinitionID();
   }
 };
 
+static_assert(alignof(ObjCCategoriesInfo) <= 4);
+static_assert(std::is_standard_layout_v<ObjCCategoriesInfo> &&
+              std::is_trivial_v<ObjCCategoriesInfo>);
+
 /// A key used when looking up entities by \ref DeclarationName.
 ///
 /// Different \ref DeclarationNames are mapped to different keys, but the
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index bea58d60d36c41c02ed926141b9bad313b239724..0a9006223dcbd589fbc75623be30d2dadbcbc5b4 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -504,12 +504,6 @@ private:
   /// = I + 1 has already been loaded.
   llvm::PagedVector<Decl *> DeclsLoaded;
 
-  using GlobalDeclMapType = ContinuousRangeMap<GlobalDeclID, ModuleFile *, 4>;
-
-  /// Mapping from global declaration IDs to the module in which the
-  /// declaration resides.
-  GlobalDeclMapType GlobalDeclMap;
-
   using FileOffset = std::pair<ModuleFile *, uint64_t>;
   using FileOffsetsTy = SmallVector<FileOffset, 2>;
   using DeclUpdateOffsetsMap = llvm::DenseMap<GlobalDeclID, FileOffsetsTy>;
@@ -592,10 +586,11 @@ private:
 
   struct FileDeclsInfo {
     ModuleFile *Mod = nullptr;
-    ArrayRef<LocalDeclID> Decls;
+    ArrayRef<serialization::unaligned_decl_id_t> Decls;
 
     FileDeclsInfo() = default;
-    FileDeclsInfo(ModuleFile *Mod, ArrayRef<LocalDeclID> Decls)
+    FileDeclsInfo(ModuleFile *Mod,
+                  ArrayRef<serialization::unaligned_decl_id_t> Decls)
         : Mod(Mod), Decls(Decls) {}
   };
 
@@ -604,11 +599,7 @@ private:
 
   /// An array of lexical contents of a declaration context, as a sequence of
   /// Decl::Kind, DeclID pairs.
-  using unaligned_decl_id_t =
-      llvm::support::detail::packed_endian_specific_integral<
-          serialization::DeclID, llvm::endianness::native,
-          llvm::support::unaligned>;
-  using LexicalContents = ArrayRef<unaligned_decl_id_t>;
+  using LexicalContents = ArrayRef<serialization::unaligned_decl_id_t>;
 
   /// Map from a DeclContext to its lexical contents.
   llvm::DenseMap<const DeclContext*, std::pair<ModuleFile*, LexicalContents>>
@@ -1489,10 +1480,11 @@ private:
                                unsigned ClientLoadCapabilities);
 
 public:
-  class ModuleDeclIterator : public llvm::iterator_adaptor_base<
-                                 ModuleDeclIterator, const LocalDeclID *,
-                                 std::random_access_iterator_tag, const Decl *,
-                                 ptrdiff_t, const Decl *, const Decl *> {
+  class ModuleDeclIterator
+      : public llvm::iterator_adaptor_base<
+            ModuleDeclIterator, const serialization::unaligned_decl_id_t *,
+            std::random_access_iterator_tag, const Decl *, ptrdiff_t,
+            const Decl *, const Decl *> {
     ASTReader *Reader = nullptr;
     ModuleFile *Mod = nullptr;
 
@@ -1500,11 +1492,11 @@ public:
     ModuleDeclIterator() : iterator_adaptor_base(nullptr) {}
 
     ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod,
-                       const LocalDeclID *Pos)
+                       const serialization::unaligned_decl_id_t *Pos)
         : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {}
 
     value_type operator*() const {
-      return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I));
+      return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I));
     }
 
     value_type operator->() const { return **this; }
@@ -1544,6 +1536,9 @@ private:
              StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const;
   void Error(llvm::Error &&Err) const;
 
+  /// Translate a \param GlobalDeclID to the index of DeclsLoaded array.
+  unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const;
+
 public:
   /// Load the AST file and validate its contents against the given
   /// Preprocessor.
@@ -1915,7 +1910,8 @@ public:
 
   /// Retrieve the module file that owns the given declaration, or NULL
   /// if the declaration is not from a module file.
-  ModuleFile *getOwningModuleFile(const Decl *D);
+  ModuleFile *getOwningModuleFile(const Decl *D) const;
+  ModuleFile *getOwningModuleFile(GlobalDeclID ID) const;
 
   /// Returns the source location for the decl \p ID.
   SourceLocation getSourceLocationForDeclID(GlobalDeclID ID);
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 992d26a8b88c12d4dabb666b8de65ce97ce884ba..56193d44dd6f33bb04718f79ec93307d9ab83d14 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -454,23 +454,11 @@ public:
   /// by the declaration ID (-1).
   const DeclOffset *DeclOffsets = nullptr;
 
-  /// Base declaration ID for declarations local to this module.
-  serialization::DeclID BaseDeclID = 0;
-
-  /// Remapping table for declaration IDs in this module.
-  ContinuousRangeMap<serialization::DeclID, int, 2> DeclRemap;
-
-  /// Mapping from the module files that this module file depends on
-  /// to the base declaration ID for that module as it is understood within this
-  /// module.
-  ///
-  /// This is effectively a reverse global-to-local mapping for declaration
-  /// IDs, so that we can interpret a true global ID (for this translation unit)
-  /// as a local ID (for this module file).
-  llvm::DenseMap<ModuleFile *, serialization::DeclID> GlobalToLocalDeclIDs;
+  /// Base declaration index in ASTReader for declarations local to this module.
+  unsigned BaseDeclIndex = 0;
 
   /// Array of file-level DeclIDs sorted by file.
-  const LocalDeclID *FileSortedDecls = nullptr;
+  const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr;
   unsigned NumFileSortedDecls = 0;
 
   /// Array of category list location information within this
diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h
index d770bc52eaf4f734fcfc817d08176af497e7c1cc..f898dab39f06d3527c901a231ed0fe67ddc66b77 100644
--- a/clang/include/clang/Serialization/ModuleManager.h
+++ b/clang/include/clang/Serialization/ModuleManager.h
@@ -45,7 +45,7 @@ namespace serialization {
 /// Manages the set of modules loaded by an AST reader.
 class ModuleManager {
   /// The chain of AST files, in the order in which we started to load
-  /// them (this order isn't really useful for anything).
+  /// them.
   SmallVector<std::unique_ptr<ModuleFile>, 2> Chain;
 
   /// The chain of non-module PCH files. The first entry is the one named
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 9dc20065a09a3318d9e5f3b372c07399ade108a0..f7b4510d7f7beb4b6e300180453385e1a84e6af4 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -363,8 +363,7 @@ public:
   ///
   /// Returns true if the directive tokens are populated for this file entry,
   /// false if not (i.e. this entry is not a file or its scan fails).
-  bool ensureDirectiveTokensArePopulated(EntryRef Entry,
-                                         const LangOptions &LangOpts);
+  bool ensureDirectiveTokensArePopulated(EntryRef Entry);
 
   /// Check whether \p Path exists. By default checks cached result of \c
   /// status(), and falls back on FS if unable to do so.
diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h
index b1bdefed7c97f1163adbe1bbed8eaabc28dca429..f71b8d67bfea4f8626933770c36a8a841bc3c8d5 100644
--- a/clang/include/clang/Tooling/Syntax/Tokens.h
+++ b/clang/include/clang/Tooling/Syntax/Tokens.h
@@ -292,9 +292,9 @@ public:
   ///     "DECL", "(", "a", ")", ";"}
   llvm::ArrayRef<syntax::Token> spelledTokens(FileID FID) const;
 
-  /// Returns the spelled Token starting at Loc, if there are no such tokens
+  /// Returns the spelled Token containing the Loc, if there are no such tokens
   /// returns nullptr.
-  const syntax::Token *spelledTokenAt(SourceLocation Loc) const;
+  const syntax::Token *spelledTokenContaining(SourceLocation Loc) const;
 
   /// Get all tokens that expand a macro in \p FID. For the following input
   ///     #define FOO B
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index bf74e56a14799c7ed1666bbb1b6ea7c315d49253..cd76b8aa271dabf115629d67c496e8750f2829b2 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -87,6 +87,7 @@
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/AArch64TargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
 #include <cassert>
@@ -13663,17 +13664,20 @@ QualType ASTContext::getCorrespondingSignedFixedPointType(QualType Ty) const {
   }
 }
 
-std::vector<std::string> ASTContext::filterFunctionTargetVersionAttrs(
-    const TargetVersionAttr *TV) const {
-  assert(TV != nullptr);
-  llvm::SmallVector<StringRef, 8> Feats;
-  std::vector<std::string> ResFeats;
-  TV->getFeatures(Feats);
-  for (auto &Feature : Feats)
-    if (Target->validateCpuSupports(Feature.str()))
-      // Use '?' to mark features that came from TargetVersion.
-      ResFeats.push_back("?" + Feature.str());
-  return ResFeats;
+// Given a list of FMV features, return a concatenated list of the
+// corresponding backend features (which may contain duplicates).
+static std::vector<std::string> getFMVBackendFeaturesFor(
+    const llvm::SmallVectorImpl<StringRef> &FMVFeatStrings) {
+  std::vector<std::string> BackendFeats;
+  for (StringRef F : FMVFeatStrings) {
+    if (auto FMVExt = llvm::AArch64::parseArchExtension(F)) {
+      SmallVector<StringRef, 8> Feats;
+      FMVExt->DependentFeatures.split(Feats, ',', -1, false);
+      for (StringRef F : Feats)
+        BackendFeats.push_back(F.str());
+    }
+  }
+  return BackendFeats;
 }
 
 ParsedTargetAttr
@@ -13708,10 +13712,12 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
 
     // Make a copy of the features as passed on the command line into the
     // beginning of the additional features from the function to override.
-    ParsedAttr.Features.insert(
-        ParsedAttr.Features.begin(),
-        Target->getTargetOpts().FeaturesAsWritten.begin(),
-        Target->getTargetOpts().FeaturesAsWritten.end());
+    // AArch64 handles command line option features in parseTargetAttr().
+    if (!Target->getTriple().isAArch64())
+      ParsedAttr.Features.insert(
+          ParsedAttr.Features.begin(),
+          Target->getTargetOpts().FeaturesAsWritten.begin(),
+          Target->getTargetOpts().FeaturesAsWritten.end());
 
     if (ParsedAttr.CPU != "" && Target->isValidCPUName(ParsedAttr.CPU))
       TargetCPU = ParsedAttr.CPU;
@@ -13732,32 +13738,31 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                     Target->getTargetOpts().FeaturesAsWritten.end());
     Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
-    std::vector<std::string> Features;
     if (Target->getTriple().isAArch64()) {
-      // TargetClones for AArch64
       llvm::SmallVector<StringRef, 8> Feats;
       TC->getFeatures(Feats, GD.getMultiVersionIndex());
-      for (StringRef Feat : Feats)
-        if (Target->validateCpuSupports(Feat.str()))
-          // Use '?' to mark features that came from AArch64 TargetClones.
-          Features.push_back("?" + Feat.str());
+      std::vector<std::string> Features = getFMVBackendFeaturesFor(Feats);
       Features.insert(Features.begin(),
                       Target->getTargetOpts().FeaturesAsWritten.begin(),
                       Target->getTargetOpts().FeaturesAsWritten.end());
+      Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
     } else {
+      std::vector<std::string> Features;
       StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
       if (VersionStr.starts_with("arch="))
         TargetCPU = VersionStr.drop_front(sizeof("arch=") - 1);
       else if (VersionStr != "default")
         Features.push_back((StringRef{"+"} + VersionStr).str());
+      Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
     }
-    Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else if (const auto *TV = FD->getAttr<TargetVersionAttr>()) {
-    std::vector<std::string> Feats = filterFunctionTargetVersionAttrs(TV);
-    Feats.insert(Feats.begin(),
-                 Target->getTargetOpts().FeaturesAsWritten.begin(),
-                 Target->getTargetOpts().FeaturesAsWritten.end());
-    Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Feats);
+    llvm::SmallVector<StringRef, 8> Feats;
+    TV->getFeatures(Feats);
+    std::vector<std::string> Features = getFMVBackendFeaturesFor(Feats);
+    Features.insert(Features.begin(),
+                    Target->getTargetOpts().FeaturesAsWritten.begin(),
+                    Target->getTargetOpts().FeaturesAsWritten.end());
+    Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else {
     FeatureMap = Target->getTargetOpts().FeatureMap;
   }
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index 3faefb54f599fb6b311607983810dc448cd5c4a9..a5d3dacfc1a84e1abfbebfb62d403b32189778fb 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -87,6 +87,7 @@ add_clang_library(clangAST
   Interp/Record.cpp
   Interp/Source.cpp
   Interp/State.cpp
+  Interp/MemberPointer.cpp
   Interp/InterpShared.cpp
   ItaniumCXXABI.cpp
   ItaniumMangle.cpp
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 1e9c879e371bc176a442d8ec0b3933c8eb9eddb0..7f1ed9c691e988e903e249d7364de7e467bd668b 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -74,18 +74,17 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context,
                          GlobalDeclID ID, std::size_t Extra) {
   // Allocate an extra 8 bytes worth of storage, which ensures that the
   // resulting pointer will still be 8-byte aligned.
-  static_assert(sizeof(unsigned) * 2 >= alignof(Decl),
-                "Decl won't be misaligned");
+  static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned");
   void *Start = Context.Allocate(Size + Extra + 8);
   void *Result = (char*)Start + 8;
 
-  unsigned *PrefixPtr = (unsigned *)Result - 2;
+  uint64_t *PrefixPtr = (uint64_t *)Result - 1;
 
-  // Zero out the first 4 bytes; this is used to store the owning module ID.
-  PrefixPtr[0] = 0;
+  *PrefixPtr = ID.get();
 
-  // Store the global declaration ID in the second 4 bytes.
-  PrefixPtr[1] = ID.get();
+  // We leave the upper 16 bits to store the module IDs. 48 bits should be
+  // sufficient to store a declaration ID.
+  assert(*PrefixPtr < llvm::maskTrailingOnes<uint64_t>(48));
 
   return Result;
 }
@@ -111,6 +110,29 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx,
   return ::operator new(Size + Extra, Ctx);
 }
 
+GlobalDeclID Decl::getGlobalID() const {
+  if (!isFromASTFile())
+    return GlobalDeclID();
+  // See the comments in `Decl::operator new` for details.
+  uint64_t ID = *((const uint64_t *)this - 1);
+  return GlobalDeclID(ID & llvm::maskTrailingOnes<uint64_t>(48));
+}
+
+unsigned Decl::getOwningModuleID() const {
+  if (!isFromASTFile())
+    return 0;
+
+  uint64_t ID = *((const uint64_t *)this - 1);
+  return ID >> 48;
+}
+
+void Decl::setOwningModuleID(unsigned ID) {
+  assert(isFromASTFile() && "Only works on a deserialized declaration");
+  uint64_t *IDAddress = (uint64_t *)this - 1;
+  *IDAddress &= llvm::maskTrailingOnes<uint64_t>(48);
+  *IDAddress |= (uint64_t)ID << 48;
+}
+
 Module *Decl::getOwningModuleSlow() const {
   assert(isFromASTFile() && "Not from AST file?");
   return getASTContext().getExternalSource()->getModule(getOwningModuleID());
@@ -2163,3 +2185,7 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C,
 
   return DD;
 }
+
+unsigned DeclIDBase::getLocalDeclIndex() const {
+  return ID & llvm::maskTrailingOnes<DeclID>(32);
+}
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 75c441293d62e2038743e7b2f218c70082a5b665..7f2c786547b9b6eaa2b9cecfd778ed51d96f356c 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -68,8 +68,8 @@ void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const {
   assert(Source && "getFromExternalSource with no external source");
 
   for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I)
-    I.setDecl(cast<NamedDecl>(Source->GetExternalDecl(
-        GlobalDeclID(reinterpret_cast<uintptr_t>(I.getDecl()) >> 2))));
+    I.setDecl(
+        cast<NamedDecl>(Source->GetExternalDecl(GlobalDeclID(I.getDeclID()))));
   Impl.Decls.setLazy(false);
 }
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index f1aa19e4409e159c2a008901306876e615b2f6e3..86fb396fabe2d942dcb3b422a92cce09363dd88c 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -15209,11 +15209,21 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       APFloat &ResI = Result.getComplexFloatImag();
       if (LHSReal) {
         assert(!RHSReal && "Cannot have two real operands for a complex op!");
-        ResR = A * C;
-        ResI = A * D;
+        ResR = A;
+        ResI = A;
+        // ResR = A * C;
+        // ResI = A * D;
+        if (!handleFloatFloatBinOp(Info, E, ResR, BO_Mul, C) ||
+            !handleFloatFloatBinOp(Info, E, ResI, BO_Mul, D))
+          return false;
       } else if (RHSReal) {
-        ResR = C * A;
-        ResI = C * B;
+        // ResR = C * A;
+        // ResI = C * B;
+        ResR = C;
+        ResI = C;
+        if (!handleFloatFloatBinOp(Info, E, ResR, BO_Mul, A) ||
+            !handleFloatFloatBinOp(Info, E, ResI, BO_Mul, B))
+          return false;
       } else {
         // In the fully general case, we need to handle NaNs and infinities
         // robustly.
@@ -15289,8 +15299,13 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       APFloat &ResR = Result.getComplexFloatReal();
       APFloat &ResI = Result.getComplexFloatImag();
       if (RHSReal) {
-        ResR = A / C;
-        ResI = B / C;
+        ResR = A;
+        ResI = B;
+        // ResR = A / C;
+        // ResI = B / C;
+        if (!handleFloatFloatBinOp(Info, E, ResR, BO_Div, C) ||
+            !handleFloatFloatBinOp(Info, E, ResI, BO_Div, C))
+          return false;
       } else {
         if (LHSReal) {
           // No real optimizations we can do here, stub out with zero.
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 3671c41ae7039c655daeb27272763160f4ef53b6..6654a27c9216893dbbaee50f8a8d2f2cd9388218 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -100,6 +100,35 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     return this->emitMemcpy(CE);
   }
 
+  case CK_DerivedToBaseMemberPointer: {
+    assert(classifyPrim(CE->getType()) == PT_MemberPtr);
+    assert(classifyPrim(SubExpr->getType()) == PT_MemberPtr);
+    const auto *FromMP = SubExpr->getType()->getAs<MemberPointerType>();
+    const auto *ToMP = CE->getType()->getAs<MemberPointerType>();
+
+    unsigned DerivedOffset = collectBaseOffset(QualType(ToMP->getClass(), 0),
+                                               QualType(FromMP->getClass(), 0));
+
+    if (!this->visit(SubExpr))
+      return false;
+
+    return this->emitGetMemberPtrBasePop(DerivedOffset, CE);
+  }
+
+  case CK_BaseToDerivedMemberPointer: {
+    assert(classifyPrim(CE) == PT_MemberPtr);
+    assert(classifyPrim(SubExpr) == PT_MemberPtr);
+    const auto *FromMP = SubExpr->getType()->getAs<MemberPointerType>();
+    const auto *ToMP = CE->getType()->getAs<MemberPointerType>();
+
+    unsigned DerivedOffset = collectBaseOffset(QualType(FromMP->getClass(), 0),
+                                               QualType(ToMP->getClass(), 0));
+
+    if (!this->visit(SubExpr))
+      return false;
+    return this->emitGetMemberPtrBasePop(-DerivedOffset, CE);
+  }
+
   case CK_UncheckedDerivedToBase:
   case CK_DerivedToBase: {
     if (!this->visit(SubExpr))
@@ -187,7 +216,8 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     return this->emitCastFloatingIntegral(*ToT, CE);
   }
 
-  case CK_NullToPointer: {
+  case CK_NullToPointer:
+  case CK_NullToMemberPointer: {
     if (DiscardResult)
       return true;
 
@@ -295,8 +325,11 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
     assert(isPtrType(*FromT));
     assert(isPtrType(*ToT));
-    if (FromT == ToT)
+    if (FromT == ToT) {
+      if (SubExpr->getType()->isVoidPointerType())
+        return this->visit(SubExpr) && this->emitVoidPtrCast(CE);
       return this->delegate(SubExpr);
+    }
 
     if (!this->visit(SubExpr))
       return false;
@@ -326,7 +359,8 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     return this->emitCast(*FromT, *ToT, CE);
   }
 
-  case CK_PointerToBoolean: {
+  case CK_PointerToBoolean:
+  case CK_MemberPointerToBoolean: {
     PrimType PtrT = classifyPrim(SubExpr->getType());
 
     // Just emit p != nullptr for this.
@@ -534,8 +568,23 @@ bool ByteCodeExprGen<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
       BO->isComparisonOp())
     return this->emitComplexComparison(LHS, RHS, BO);
 
-  if (BO->isPtrMemOp())
-    return this->visit(RHS);
+  if (BO->isPtrMemOp()) {
+    if (!this->visit(LHS))
+      return false;
+
+    if (!this->visit(RHS))
+      return false;
+
+    if (!this->emitToMemberPtr(BO))
+      return false;
+
+    if (classifyPrim(BO) == PT_MemberPtr)
+      return true;
+
+    if (!this->emitCastMemberPtrPtr(BO))
+      return false;
+    return DiscardResult ? this->emitPopPtr(BO) : true;
+  }
 
   // Typecheck the args.
   std::optional<PrimType> LT = classify(LHS->getType());
@@ -1058,14 +1107,9 @@ bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
       if (!this->visit(Init))
         return false;
 
-      if (FieldToInit->isBitField()) {
-        if (!this->emitInitBitField(T, FieldToInit, E))
-          return false;
-      } else {
-        if (!this->emitInitField(T, FieldToInit->Offset, E))
-          return false;
-      }
-      return this->emitPopPtr(E);
+      if (FieldToInit->isBitField())
+        return this->emitInitBitField(T, FieldToInit, E);
+      return this->emitInitField(T, FieldToInit->Offset, E);
     };
 
     auto initCompositeField = [=](const Record::Field *FieldToInit,
@@ -1101,9 +1145,6 @@ bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
         else
           FToInit = cast<CXXParenListInitExpr>(E)->getInitializedFieldInUnion();
 
-        if (!this->emitDupPtr(E))
-          return false;
-
         const Record::Field *FieldToInit = R->getField(FToInit);
         if (std::optional<PrimType> T = classify(Init)) {
           if (!initPrimitiveField(FieldToInit, Init, *T))
@@ -1123,8 +1164,6 @@ bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
       while (InitIndex < R->getNumFields() &&
              R->getField(InitIndex)->Decl->isUnnamedBitField())
         ++InitIndex;
-      if (!this->emitDupPtr(E))
-        return false;
 
       if (std::optional<PrimType> T = classify(Init)) {
         const Record::Field *FieldToInit = R->getField(InitIndex);
@@ -1134,7 +1173,7 @@ bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
       } else {
         // Initializer for a direct base class.
         if (const Record::Base *B = R->getBase(Init->getType())) {
-          if (!this->emitGetPtrBasePop(B->Offset, Init))
+          if (!this->emitGetPtrBase(B->Offset, Init))
             return false;
 
           if (!this->visitInitializer(Init))
@@ -1467,7 +1506,7 @@ bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
     // Leave a pointer to the field on the stack.
     if (F->Decl->getType()->isReferenceType())
       return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue();
-    return this->emitGetPtrField(F->Offset, E) && maybeLoadValue();
+    return this->emitGetPtrFieldPop(F->Offset, E) && maybeLoadValue();
   }
 
   return false;
@@ -1798,6 +1837,9 @@ bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
   std::optional<PrimType> RT = classify(RHS->getType());
   std::optional<PrimType> ResultT = classify(E->getType());
 
+  if (!Ctx.getLangOpts().CPlusPlus14)
+    return this->visit(RHS) && this->visit(LHS) && this->emitError(E);
+
   if (!LT || !RT || !ResultT || !LHSComputationT)
     return false;
 
@@ -2101,9 +2143,6 @@ bool ByteCodeExprGen<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
       if (!this->emitInitField(*T, F.Offset, E))
         return false;
     } else {
-      if (!this->emitDupPtr(E))
-        return false;
-
       if (!this->emitGetPtrField(F.Offset, E))
         return false;
 
@@ -2773,6 +2812,8 @@ bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
     return this->emitNullPtr(nullptr, E);
   case PT_FnPtr:
     return this->emitNullFnPtr(nullptr, E);
+  case PT_MemberPtr:
+    return this->emitNullMemberPtr(nullptr, E);
   case PT_Float: {
     return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
   }
@@ -2798,9 +2839,6 @@ bool ByteCodeExprGen<Emitter>::visitZeroRecordInitializer(const Record *R,
       continue;
     }
 
-    // TODO: Add GetPtrFieldPop and get rid of this dup.
-    if (!this->emitDupPtr(E))
-      return false;
     if (!this->emitGetPtrField(Field.Offset, E))
       return false;
 
@@ -2875,6 +2913,7 @@ bool ByteCodeExprGen<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
     return this->emitConstBool(Value, E);
   case PT_Ptr:
   case PT_FnPtr:
+  case PT_MemberPtr:
   case PT_Float:
   case PT_IntAP:
   case PT_IntAPS:
@@ -3169,9 +3208,18 @@ bool ByteCodeExprGen<Emitter>::visitAPValue(const APValue &Val,
     return this->emitConst(Val.getInt(), ValType, E);
 
   if (Val.isLValue()) {
+    if (Val.isNullPointer())
+      return this->emitNull(ValType, nullptr, E);
     APValue::LValueBase Base = Val.getLValueBase();
     if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
       return this->visit(BaseExpr);
+    else if (const auto *VD = Base.dyn_cast<const ValueDecl *>()) {
+      return this->visitDeclRef(VD, E);
+    }
+  } else if (Val.isMemberPointer()) {
+    if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl())
+      return this->emitGetMemberPtr(MemberDecl, E);
+    return this->emitNullMemberPtr(nullptr, E);
   }
 
   return false;
@@ -3180,15 +3228,15 @@ bool ByteCodeExprGen<Emitter>::visitAPValue(const APValue &Val,
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::visitAPValueInitializer(const APValue &Val,
                                                        const Expr *E) {
+
   if (Val.isStruct()) {
     const Record *R = this->getRecord(E->getType());
     assert(R);
-
     for (unsigned I = 0, N = Val.getStructNumFields(); I != N; ++I) {
       const APValue &F = Val.getStructField(I);
       const Record::Field *RF = R->getField(I);
 
-      if (F.isInt()) {
+      if (F.isInt() || F.isLValue() || F.isMemberPointer()) {
         PrimType T = classifyPrim(RF->Decl->getType());
         if (!this->visitAPValue(F, T, E))
           return false;
@@ -3200,8 +3248,6 @@ bool ByteCodeExprGen<Emitter>::visitAPValueInitializer(const APValue &Val,
         PrimType ElemT = classifyPrim(ArrType->getElementType());
         assert(ArrType);
 
-        if (!this->emitDupPtr(E))
-          return false;
         if (!this->emitGetPtrField(RF->Offset, E))
           return false;
 
@@ -3214,11 +3260,28 @@ bool ByteCodeExprGen<Emitter>::visitAPValueInitializer(const APValue &Val,
 
         if (!this->emitPopPtr(E))
           return false;
+      } else if (F.isStruct() || F.isUnion()) {
+        if (!this->emitGetPtrField(RF->Offset, E))
+          return false;
+        if (!this->visitAPValueInitializer(F, E))
+          return false;
+        if (!this->emitPopPtr(E))
+          return false;
       } else {
         assert(false && "I don't think this should be possible");
       }
     }
     return true;
+  } else if (Val.isUnion()) {
+    const FieldDecl *UnionField = Val.getUnionField();
+    const Record *R = this->getRecord(UnionField->getParent());
+    assert(R);
+    const APValue &F = Val.getUnionValue();
+    const Record::Field *RF = R->getField(UnionField);
+    PrimType T = classifyPrim(RF->Decl->getType());
+    if (!this->visitAPValue(F, T, E))
+      return false;
+    return this->emitInitElem(T, 0, E);
   }
   // TODO: Other types.
 
@@ -3308,10 +3371,27 @@ bool ByteCodeExprGen<Emitter>::VisitCallExpr(const CallExpr *E) {
     }
   }
 
+  std::optional<unsigned> CalleeOffset;
   // Add the (optional, implicit) This pointer.
   if (const auto *MC = dyn_cast<CXXMemberCallExpr>(E)) {
-    if (!this->visit(MC->getImplicitObjectArgument()))
+    if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) {
+      // If we end up creating a CallPtr op for this, we need the base of the
+      // member pointer as the instance pointer, and later extract the function
+      // decl as the function pointer.
+      const Expr *Callee = E->getCallee();
+      CalleeOffset =
+          this->allocateLocalPrimitive(Callee, PT_MemberPtr, true, false);
+      if (!this->visit(Callee))
+        return false;
+      if (!this->emitSetLocal(PT_MemberPtr, *CalleeOffset, E))
+        return false;
+      if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
+        return false;
+      if (!this->emitGetMemberPtrBase(E))
+        return false;
+    } else if (!this->visit(MC->getImplicitObjectArgument())) {
       return false;
+    }
   }
 
   llvm::BitVector NonNullArgs = collectNonNullArgs(FuncDecl, Args);
@@ -3380,11 +3460,22 @@ bool ByteCodeExprGen<Emitter>::VisitCallExpr(const CallExpr *E) {
     for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
       ArgSize += align(primSize(classify(E->getArg(I)).value_or(PT_Ptr)));
 
-    if (!this->visit(E->getCallee()))
-      return false;
+    // Get the callee, either from a member pointer saved in CalleeOffset,
+    // or by just visiting the Callee expr.
+    if (CalleeOffset) {
+      if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
+        return false;
+      if (!this->emitGetMemberPtrDecl(E))
+        return false;
+      if (!this->emitCallPtr(ArgSize, E, E))
+        return false;
+    } else {
+      if (!this->visit(E->getCallee()))
+        return false;
 
-    if (!this->emitCallPtr(ArgSize, E, E))
-      return false;
+      if (!this->emitCallPtr(ArgSize, E, E))
+        return false;
+    }
   }
 
   // Cleanup for discarded return values.
@@ -3623,6 +3714,11 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       return false;
     return DiscardResult ? this->emitPop(*T, E) : true;
   case UO_AddrOf: // &x
+    if (E->getType()->isMemberPointerType()) {
+      // C++11 [expr.unary.op]p3 has very strict rules on how the address of a
+      // member can be formed.
+      return this->emitGetMemberPtr(cast<DeclRefExpr>(SubExpr)->getDecl(), E);
+    }
     // We should already have a pointer when we get here.
     return this->delegate(SubExpr);
   case UO_Deref:  // *x
@@ -3737,6 +3833,21 @@ bool ByteCodeExprGen<Emitter>::VisitComplexUnaryOperator(
     // we sometimes have to do the lvalue-to-rvalue conversion here manually.
     return this->emitArrayElemPop(classifyPrim(E->getType()), 1, E);
 
+  case UO_Not: // ~x
+    if (!this->visit(SubExpr))
+      return false;
+    // Negate the imaginary component.
+    if (!this->emitArrayElem(ElemT, 1, E))
+      return false;
+    if (!this->emitNeg(ElemT, E))
+      return false;
+    if (!this->emitInitElem(ElemT, 1, E))
+      return false;
+    return DiscardResult ? this->emitPopPtr(E) : true;
+
+  case UO_Extension:
+    return this->delegate(SubExpr);
+
   default:
     return this->emitInvalid(E);
   }
@@ -3745,12 +3856,10 @@ bool ByteCodeExprGen<Emitter>::VisitComplexUnaryOperator(
 }
 
 template <class Emitter>
-bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
+bool ByteCodeExprGen<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
   if (DiscardResult)
     return true;
 
-  const auto *D = E->getDecl();
-
   if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
     return this->emitConst(ECD->getInitVal(), E);
   } else if (const auto *BD = dyn_cast<BindingDecl>(D)) {
@@ -3806,6 +3915,14 @@ bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
     if (IsPtr)
       return this->emitGetThisFieldPtr(Offset, E);
     return this->emitGetPtrThisField(Offset, E);
+  } else if (const auto *DRE = dyn_cast<DeclRefExpr>(E);
+             DRE && DRE->refersToEnclosingVariableOrCapture()) {
+    if (const auto *VD = dyn_cast<VarDecl>(D); VD && VD->isInitCapture()) {
+      if (!this->visitVarDecl(cast<VarDecl>(D)))
+        return false;
+      // Retry.
+      return this->visitDeclRef(D, E);
+    }
   }
 
   // Try to lazily visit (or emit dummy pointers for) declarations
@@ -3818,7 +3935,7 @@ bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
         if (!this->visitVarDecl(VD))
           return false;
         // Retry.
-        return this->VisitDeclRefExpr(E);
+        return this->visitDeclRef(VD, E);
       }
     }
   } else {
@@ -3828,7 +3945,7 @@ bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
       if (!this->visitVarDecl(VD))
         return false;
       // Retry.
-      return this->VisitDeclRefExpr(E);
+      return this->visitDeclRef(VD, E);
     }
   }
 
@@ -3845,7 +3962,15 @@ bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
     return true;
   }
 
-  return this->emitInvalidDeclRef(E, E);
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+    return this->emitInvalidDeclRef(DRE, E);
+  return false;
+}
+
+template <class Emitter>
+bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
+  const auto *D = E->getDecl();
+  return this->visitDeclRef(D, E);
 }
 
 template <class Emitter>
@@ -4085,8 +4210,6 @@ bool ByteCodeExprGen<Emitter>::emitRecordDestruction(const Record *R) {
   for (const Record::Field &Field : llvm::reverse(R->fields())) {
     const Descriptor *D = Field.Desc;
     if (!D->isPrimitive() && !D->isPrimitiveArray()) {
-      if (!this->emitDupPtr(SourceInfo{}))
-        return false;
       if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
         return false;
       if (!this->emitDestruction(D))
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index 44c495240289f8b7251c776120762c2670aaceea..7ab14b6ab383e84d03fa88903b4c473debff0c91 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -189,6 +189,8 @@ protected:
   /// Visit an APValue.
   bool visitAPValue(const APValue &Val, PrimType ValType, const Expr *E);
   bool visitAPValueInitializer(const APValue &Val, const Expr *E);
+  /// Visit the given decl as if we have a reference to it.
+  bool visitDeclRef(const ValueDecl *D, const Expr *E);
 
   /// Visits an expression and converts it to a boolean.
   bool visitBool(const Expr *E);
diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp
index b0b22b059b77ad03fe671833da769bba72c91fb6..98d1837204ebc4eda7cedde5d9794137731e97bd 100644
--- a/clang/lib/AST/Interp/Context.cpp
+++ b/clang/lib/AST/Interp/Context.cpp
@@ -163,8 +163,12 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (T->isFloatingType())
     return PT_Float;
 
+  if (T->isSpecificBuiltinType(BuiltinType::BoundMember) ||
+      T->isMemberPointerType())
+    return PT_MemberPtr;
+
   if (T->isFunctionPointerType() || T->isFunctionReferenceType() ||
-      T->isFunctionType() || T->isSpecificBuiltinType(BuiltinType::BoundMember))
+      T->isFunctionType())
     return PT_FnPtr;
 
   if (T->isReferenceType() || T->isPointerType() ||
@@ -177,9 +181,6 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (const auto *DT = dyn_cast<DecltypeType>(T))
     return classify(DT->getUnderlyingType());
 
-  if (const auto *DT = dyn_cast<MemberPointerType>(T))
-    return classify(DT->getPointeeType());
-
   return std::nullopt;
 }
 
@@ -292,10 +293,12 @@ unsigned Context::collectBaseOffset(const RecordDecl *BaseDecl,
     }
     if (CurDecl == FinalDecl)
       break;
-
-    // break;
   }
 
   assert(OffsetSum > 0);
   return OffsetSum;
 }
+
+const Record *Context::getRecord(const RecordDecl *D) const {
+  return P->getOrCreateRecord(D);
+}
diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/Interp/Context.h
index 360e9499d0844071e0ffe9b3c857c93a9800ada9..c78dc9a2a471eb7af44371a139d525e2a1df4460 100644
--- a/clang/lib/AST/Interp/Context.h
+++ b/clang/lib/AST/Interp/Context.h
@@ -107,6 +107,8 @@ public:
   unsigned collectBaseOffset(const RecordDecl *BaseDecl,
                              const RecordDecl *DerivedDecl) const;
 
+  const Record *getRecord(const RecordDecl *D) const;
+
 private:
   /// Runs a function.
   bool Run(State &Parent, const Function *Func, APValue &Result);
diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp
index 746b765ca421679b33a437b4ba1913c385efad26..d20ab1340c89039736c31a01d8827a37973b0745 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/Interp/Descriptor.cpp
@@ -11,6 +11,7 @@
 #include "Floating.h"
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
+#include "MemberPointer.h"
 #include "Pointer.h"
 #include "PrimType.h"
 #include "Record.h"
diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp
index 3f8a92ed2f0bde33104cac48998ec017c87f40b1..0ab84d159c58b733c8dcaac4343d12a611e0cf89 100644
--- a/clang/lib/AST/Interp/Disasm.cpp
+++ b/clang/lib/AST/Interp/Disasm.cpp
@@ -19,6 +19,7 @@
 #include "Integral.h"
 #include "IntegralAP.h"
 #include "InterpFrame.h"
+#include "MemberPointer.h"
 #include "Opcode.h"
 #include "PrimType.h"
 #include "Program.h"
@@ -122,6 +123,8 @@ static const char *primTypeToString(PrimType T) {
     return "Ptr";
   case PT_FnPtr:
     return "FnPtr";
+  case PT_MemberPtr:
+    return "MemberPtr";
   }
   llvm_unreachable("Unhandled PrimType");
 }
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp
index 6d8aa3f20f01f51dd9e2b2f2e9161a5ad515161a..025b46b3d7886af4e75d8d80646244685da7fceb 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/Interp/EvalEmitter.cpp
@@ -40,7 +40,8 @@ void EvalEmitter::cleanup() { S.cleanup(); }
 EvaluationResult EvalEmitter::interpretExpr(const Expr *E,
                                             bool ConvertResultToRValue) {
   S.setEvalLocation(E->getExprLoc());
-  this->ConvertResultToRValue = ConvertResultToRValue;
+  this->ConvertResultToRValue = ConvertResultToRValue && !isa<ConstantExpr>(E);
+  this->CheckFullyInitialized = isa<ConstantExpr>(E);
   EvalResult.setSource(E);
 
   if (!this->visitExpr(E)) {
@@ -55,10 +56,14 @@ EvaluationResult EvalEmitter::interpretExpr(const Expr *E,
 EvaluationResult EvalEmitter::interpretDecl(const VarDecl *VD,
                                             bool CheckFullyInitialized) {
   this->CheckFullyInitialized = CheckFullyInitialized;
-  this->ConvertResultToRValue =
-      VD->getAnyInitializer() &&
-      (VD->getAnyInitializer()->getType()->isAnyComplexType() ||
-       VD->getAnyInitializer()->getType()->isVectorType());
+
+  if (const Expr *Init = VD->getAnyInitializer()) {
+    QualType T = VD->getType();
+    this->ConvertResultToRValue = !Init->isGLValue() && !T->isPointerType() &&
+                                  !T->isObjCObjectPointerType();
+  } else
+    this->ConvertResultToRValue = false;
+
   EvalResult.setSource(VD);
 
   if (!this->visitDecl(VD) && EvalResult.empty())
@@ -137,6 +142,10 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
     return true;
 
   const Pointer &Ptr = S.Stk.pop<Pointer>();
+
+  if (CheckFullyInitialized && !EvalResult.checkFullyInitialized(S, Ptr))
+    return false;
+
   // Implicitly convert lvalue to rvalue, if requested.
   if (ConvertResultToRValue) {
     if (std::optional<APValue> V = Ptr.toRValue(Ctx)) {
@@ -145,17 +154,7 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
       return false;
     }
   } else {
-    if (CheckFullyInitialized) {
-      if (!EvalResult.checkFullyInitialized(S, Ptr))
-        return false;
-
-      std::optional<APValue> RValueResult = Ptr.toRValue(Ctx);
-      if (!RValueResult)
-        return false;
-      EvalResult.setValue(*RValueResult);
-    } else {
-      EvalResult.setValue(Ptr.toAPValue());
-    }
+    EvalResult.setValue(Ptr.toAPValue());
   }
 
   return true;
@@ -175,6 +174,10 @@ bool EvalEmitter::emitRetVoid(const SourceInfo &Info) {
 
 bool EvalEmitter::emitRetValue(const SourceInfo &Info) {
   const auto &Ptr = S.Stk.pop<Pointer>();
+
+  if (CheckFullyInitialized && !EvalResult.checkFullyInitialized(S, Ptr))
+    return false;
+
   if (std::optional<APValue> APV = Ptr.toRValue(S.getCtx())) {
     EvalResult.setValue(*APV);
     return true;
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp
index c04b0fc0a11212b2124c75f673d6261c6f611d0a..387e3dc88bff26f5d2135871038a7f1311efdd5c 100644
--- a/clang/lib/AST/Interp/EvaluationResult.cpp
+++ b/clang/lib/AST/Interp/EvaluationResult.cpp
@@ -141,21 +141,22 @@ bool EvaluationResult::checkFullyInitialized(InterpState &S,
                                              const Pointer &Ptr) const {
   assert(Source);
   assert(empty());
-
-  // Our Source must be a VarDecl.
-  const Decl *SourceDecl = Source.dyn_cast<const Decl *>();
-  assert(SourceDecl);
-  const auto *VD = cast<VarDecl>(SourceDecl);
-  assert(VD->getType()->isRecordType() || VD->getType()->isArrayType());
-  SourceLocation InitLoc = VD->getAnyInitializer()->getExprLoc();
-
   assert(!Ptr.isZero());
 
+  SourceLocation InitLoc;
+  if (const auto *D = Source.dyn_cast<const Decl *>())
+    InitLoc = cast<VarDecl>(D)->getAnyInitializer()->getExprLoc();
+  else if (const auto *E = Source.dyn_cast<const Expr *>())
+    InitLoc = E->getExprLoc();
+
   if (const Record *R = Ptr.getRecord())
     return CheckFieldsInitialized(S, InitLoc, Ptr, R);
-  const auto *CAT =
-      cast<ConstantArrayType>(Ptr.getType()->getAsArrayTypeUnsafe());
-  return CheckArrayInitialized(S, InitLoc, Ptr, CAT);
+
+  if (const auto *CAT = dyn_cast_if_present<ConstantArrayType>(
+          Ptr.getType()->getAsArrayTypeUnsafe()))
+    return CheckArrayInitialized(S, InitLoc, Ptr, CAT);
+
+  return true;
 }
 
 } // namespace interp
diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/Interp/Function.cpp
index 1d04998d5dd158815a51843c43983e46aad9fa73..00f5a1fced531a981996c7caa1842f9ab7f19f67 100644
--- a/clang/lib/AST/Interp/Function.cpp
+++ b/clang/lib/AST/Interp/Function.cpp
@@ -40,7 +40,8 @@ SourceInfo Function::getSource(CodePtr PC) const {
   unsigned Offset = PC - getCodeBegin();
   using Elem = std::pair<unsigned, SourceInfo>;
   auto It = llvm::lower_bound(SrcMap, Elem{Offset, {}}, llvm::less_first());
-  assert(It != SrcMap.end());
+  if (It == SrcMap.end())
+    return SrcMap.back().second;
   return It->second;
 }
 
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 145fa65791da207a2534d04c9ea9085abdadac52..49015b1dd63d33b6d75b912141c3ab6e61f037bb 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -373,6 +373,26 @@ bool CheckSubobject(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return false;
 }
 
+bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                   uint32_t Offset) {
+  uint32_t MinOffset = Ptr.getDeclDesc()->getMetadataSize();
+  uint32_t PtrOffset = Ptr.getByteOffset();
+
+  // We subtract Offset from PtrOffset. The result must be at least
+  // MinOffset.
+  if (Offset < PtrOffset && (PtrOffset - Offset) >= MinOffset)
+    return true;
+
+  const auto *E = cast<CastExpr>(S.Current->getExpr(OpPC));
+  QualType TargetQT = E->getType()->getPointeeType();
+  QualType MostDerivedQT = Ptr.getDeclPtr().getType();
+
+  S.CCEDiag(E, diag::note_constexpr_invalid_downcast)
+      << MostDerivedQT << TargetQT;
+
+  return false;
+}
+
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   assert(Ptr.isLive() && "Pointer is not live");
   if (!Ptr.isConst())
@@ -493,10 +513,12 @@ bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
 bool CheckInvoke(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   if (!CheckLive(S, OpPC, Ptr, AK_MemberCall))
     return false;
-  if (!CheckExtern(S, OpPC, Ptr))
-    return false;
-  if (!CheckRange(S, OpPC, Ptr, AK_MemberCall))
-    return false;
+  if (!Ptr.isDummy()) {
+    if (!CheckExtern(S, OpPC, Ptr))
+      return false;
+    if (!CheckRange(S, OpPC, Ptr, AK_MemberCall))
+      return false;
+  }
   return true;
 }
 
@@ -516,7 +538,7 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
     return false;
   }
 
-  if (!F->isConstexpr()) {
+  if (!F->isConstexpr() || !F->hasBody()) {
     const SourceLocation &Loc = S.Current->getLocation(OpPC);
     if (S.getLangOpts().CPlusPlus11) {
       const FunctionDecl *DiagDecl = F->getDecl();
@@ -550,9 +572,10 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
             S.checkingPotentialConstantExpression())
           return false;
 
-        // If the declaration is defined _and_ declared 'constexpr', the below
-        // diagnostic doesn't add anything useful.
-        if (DiagDecl->isDefined() && DiagDecl->isConstexpr())
+        // If the declaration is defined, declared 'constexpr' _and_ has a body,
+        // the below diagnostic doesn't add anything useful.
+        if (DiagDecl->isDefined() && DiagDecl->isConstexpr() &&
+            DiagDecl->hasBody())
           return false;
 
         S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index eca1792e64718e024dfff24a9cd73a176272d049..0ad710c5ec1afc93a305ef72d227f4ced5c4916b 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -20,6 +20,7 @@
 #include "InterpFrame.h"
 #include "InterpStack.h"
 #include "InterpState.h"
+#include "MemberPointer.h"
 #include "Opcode.h"
 #include "PrimType.h"
 #include "Program.h"
@@ -75,6 +76,11 @@ bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 bool CheckSubobject(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                     CheckSubobjectKind CSK);
 
+/// Checks if the dowcast using the given offset is possible with the given
+/// pointer.
+bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                   uint32_t Offset);
+
 /// Checks if a pointer points to const storage.
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
 
@@ -725,6 +731,9 @@ using CompareFn = llvm::function_ref<bool(ComparisonCategoryResult)>;
 
 template <typename T>
 bool CmpHelper(InterpState &S, CodePtr OpPC, CompareFn Fn) {
+  assert((!std::is_same_v<T, MemberPointer>) &&
+         "Non-equality comparisons on member pointer types should already be "
+         "rejected in Sema.");
   using BoolT = PrimConv<PT_Bool>::T;
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
@@ -834,6 +843,47 @@ inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   }
 }
 
+template <>
+inline bool CmpHelperEQ<MemberPointer>(InterpState &S, CodePtr OpPC,
+                                       CompareFn Fn) {
+  const auto &RHS = S.Stk.pop<MemberPointer>();
+  const auto &LHS = S.Stk.pop<MemberPointer>();
+
+  // If either operand is a pointer to a weak function, the comparison is not
+  // constant.
+  for (const auto &MP : {LHS, RHS}) {
+    if (const CXXMethodDecl *MD = MP.getMemberFunction(); MD && MD->isWeak()) {
+      const SourceInfo &Loc = S.Current->getSource(OpPC);
+      S.FFDiag(Loc, diag::note_constexpr_mem_pointer_weak_comparison) << MD;
+      return false;
+    }
+  }
+
+  // C++11 [expr.eq]p2:
+  //   If both operands are null, they compare equal. Otherwise if only one is
+  //   null, they compare unequal.
+  if (LHS.isZero() && RHS.isZero()) {
+    S.Stk.push<Boolean>(Fn(ComparisonCategoryResult::Equal));
+    return true;
+  }
+  if (LHS.isZero() || RHS.isZero()) {
+    S.Stk.push<Boolean>(Fn(ComparisonCategoryResult::Unordered));
+    return true;
+  }
+
+  // We cannot compare against virtual declarations at compile time.
+  for (const auto &MP : {LHS, RHS}) {
+    if (const CXXMethodDecl *MD = MP.getMemberFunction();
+        MD && MD->isVirtual()) {
+      const SourceInfo &Loc = S.Current->getSource(OpPC);
+      S.CCEDiag(Loc, diag::note_constexpr_compare_virtual_mem_ptr) << MD;
+    }
+  }
+
+  S.Stk.push<Boolean>(Boolean::from(Fn(LHS.compare(RHS))));
+  return true;
+}
+
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool EQ(InterpState &S, CodePtr OpPC) {
   return CmpHelperEQ<T>(S, OpPC, [](ComparisonCategoryResult R) {
@@ -1233,9 +1283,32 @@ inline bool GetPtrGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   return true;
 }
 
-/// 1) Pops a Pointer from the stack
+/// 1) Peeks a Pointer
 /// 2) Pushes Pointer.atField(Off) on the stack
 inline bool GetPtrField(InterpState &S, CodePtr OpPC, uint32_t Off) {
+  const Pointer &Ptr = S.Stk.peek<Pointer>();
+
+  if (S.getLangOpts().CPlusPlus && S.inConstantContext() &&
+      !CheckNull(S, OpPC, Ptr, CSK_Field))
+    return false;
+
+  if (!CheckExtern(S, OpPC, Ptr))
+    return false;
+  if (!CheckRange(S, OpPC, Ptr, CSK_Field))
+    return false;
+  if (!CheckArray(S, OpPC, Ptr))
+    return false;
+  if (!CheckSubobject(S, OpPC, Ptr, CSK_Field))
+    return false;
+
+  if (Ptr.isBlockPointer() && Off > Ptr.block()->getSize())
+    return false;
+
+  S.Stk.push<Pointer>(Ptr.atField(Off));
+  return true;
+}
+
+inline bool GetPtrFieldPop(InterpState &S, CodePtr OpPC, uint32_t Off) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   if (S.getLangOpts().CPlusPlus && S.inConstantContext() &&
@@ -1300,6 +1373,9 @@ inline bool GetPtrDerivedPop(InterpState &S, CodePtr OpPC, uint32_t Off) {
     return false;
   if (!CheckSubobject(S, OpPC, Ptr, CSK_Derived))
     return false;
+  if (!CheckDowncast(S, OpPC, Ptr, Off))
+    return false;
+
   S.Stk.push<Pointer>(Ptr.atFieldSub(Off));
   return true;
 }
@@ -1324,6 +1400,12 @@ inline bool GetPtrBasePop(InterpState &S, CodePtr OpPC, uint32_t Off) {
   return true;
 }
 
+inline bool GetMemberPtrBasePop(InterpState &S, CodePtr OpPC, int32_t Off) {
+  const auto &Ptr = S.Stk.pop<MemberPointer>();
+  S.Stk.push<MemberPointer>(Ptr.atInstanceBase(Off));
+  return true;
+}
+
 inline bool GetPtrThisBase(InterpState &S, CodePtr OpPC, uint32_t Off) {
   if (S.checkingPotentialConstantExpression())
     return false;
@@ -1532,6 +1614,24 @@ inline bool Memcpy(InterpState &S, CodePtr OpPC) {
   return DoMemcpy(S, OpPC, Src, Dest);
 }
 
+inline bool ToMemberPtr(InterpState &S, CodePtr OpPC) {
+  const auto &Member = S.Stk.pop<MemberPointer>();
+  const auto &Base = S.Stk.pop<Pointer>();
+
+  S.Stk.push<MemberPointer>(Member.takeInstance(Base));
+  return true;
+}
+
+inline bool CastMemberPtrPtr(InterpState &S, CodePtr OpPC) {
+  const auto &MP = S.Stk.pop<MemberPointer>();
+
+  if (std::optional<Pointer> Ptr = MP.toPointer(S.Ctx)) {
+    S.Stk.push<Pointer>(*Ptr);
+    return true;
+  }
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // AddOffset, SubOffset
 //===----------------------------------------------------------------------===//
@@ -1696,8 +1796,10 @@ inline bool SubPtr(InterpState &S, CodePtr OpPC) {
     return true;
   }
 
-  T A = T::from(LHS.getIndex());
-  T B = T::from(RHS.getIndex());
+  T A = LHS.isElementPastEnd() ? T::from(LHS.getNumElems())
+                               : T::from(LHS.getIndex());
+  T B = RHS.isElementPastEnd() ? T::from(RHS.getNumElems())
+                               : T::from(RHS.getIndex());
   return AddSubMulHelper<T, T::sub, std::minus>(S, OpPC, A.bitWidth(), A, B);
 }
 
@@ -1835,6 +1937,9 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastPointerIntegral(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
+  if (Ptr.isDummy())
+    return false;
+
   const SourceInfo &E = S.Current->getSource(OpPC);
   S.CCEDiag(E, diag::note_constexpr_invalid_cast)
       << 2 << S.getLangOpts().CPlusPlus << S.Current->getRange(OpPC);
@@ -1847,6 +1952,9 @@ static inline bool CastPointerIntegralAP(InterpState &S, CodePtr OpPC,
                                          uint32_t BitWidth) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
+  if (Ptr.isDummy())
+    return false;
+
   const SourceInfo &E = S.Current->getSource(OpPC);
   S.CCEDiag(E, diag::note_constexpr_invalid_cast)
       << 2 << S.getLangOpts().CPlusPlus << S.Current->getRange(OpPC);
@@ -1860,6 +1968,9 @@ static inline bool CastPointerIntegralAPS(InterpState &S, CodePtr OpPC,
                                           uint32_t BitWidth) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
+  if (Ptr.isDummy())
+    return false;
+
   const SourceInfo &E = S.Current->getSource(OpPC);
   S.CCEDiag(E, diag::note_constexpr_invalid_cast)
       << 2 << S.getLangOpts().CPlusPlus << S.Current->getRange(OpPC);
@@ -1869,6 +1980,13 @@ static inline bool CastPointerIntegralAPS(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static inline bool VoidPtrCast(InterpState &S, CodePtr OpPC) {
+  const SourceInfo &E = S.Current->getSource(OpPC);
+  S.CCEDiag(E, diag::note_constexpr_invalid_cast)
+      << 2 << S.getLangOpts().CPlusPlus << S.Current->getRange(OpPC);
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Zero, Nullptr
 //===----------------------------------------------------------------------===//
@@ -2115,7 +2233,7 @@ inline bool ArrayDecay(InterpState &S, CodePtr OpPC) {
   if (!CheckRange(S, OpPC, Ptr, CSK_ArrayToPointer))
     return false;
 
-  if (!Ptr.isUnknownSizeArray() || Ptr.isDummy()) {
+  if (Ptr.isRoot() || !Ptr.isUnknownSizeArray() || Ptr.isDummy()) {
     S.Stk.push<Pointer>(Ptr.atIndex(0));
     return true;
   }
@@ -2329,6 +2447,28 @@ inline bool GetIntPtr(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
   return true;
 }
 
+inline bool GetMemberPtr(InterpState &S, CodePtr OpPC, const Decl *D) {
+  S.Stk.push<MemberPointer>(D);
+  return true;
+}
+
+inline bool GetMemberPtrBase(InterpState &S, CodePtr OpPC) {
+  const auto &MP = S.Stk.pop<MemberPointer>();
+
+  S.Stk.push<Pointer>(MP.getBase());
+  return true;
+}
+
+inline bool GetMemberPtrDecl(InterpState &S, CodePtr OpPC) {
+  const auto &MP = S.Stk.pop<MemberPointer>();
+
+  const auto *FD = cast<FunctionDecl>(MP.getDecl());
+  const auto *Func = S.getContext().getOrCreateFunction(FD);
+
+  S.Stk.push<FunctionPointer>(Func);
+  return true;
+}
+
 /// Just emit a diagnostic. The expression that caused emission of this
 /// op is not valid in a constant context.
 inline bool Invalid(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/Interp/InterpFrame.cpp
index 51b0bd5c155159a43fd5c37e2bc1611e6359e9b2..54ccf9034c7a7dd2ff63842c7d6f27b5f35234c9 100644
--- a/clang/lib/AST/Interp/InterpFrame.cpp
+++ b/clang/lib/AST/Interp/InterpFrame.cpp
@@ -12,6 +12,7 @@
 #include "Function.h"
 #include "InterpStack.h"
 #include "InterpState.h"
+#include "MemberPointer.h"
 #include "Pointer.h"
 #include "PrimType.h"
 #include "Program.h"
diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/Interp/InterpStack.cpp
index 91fe40feb7671b6b92c3a6be763a6d4d332723f3..c7024740d322ef22d94ffc939f9ac05c9c723e78 100644
--- a/clang/lib/AST/Interp/InterpStack.cpp
+++ b/clang/lib/AST/Interp/InterpStack.cpp
@@ -10,6 +10,7 @@
 #include "Boolean.h"
 #include "Floating.h"
 #include "Integral.h"
+#include "MemberPointer.h"
 #include "Pointer.h"
 #include <cassert>
 #include <cstdlib>
diff --git a/clang/lib/AST/Interp/InterpStack.h b/clang/lib/AST/Interp/InterpStack.h
index 3fd0f63c781fc70d8623dce40f2b52d1563ab663..9d85503b851be15a47001980673c6229e2010354 100644
--- a/clang/lib/AST/Interp/InterpStack.h
+++ b/clang/lib/AST/Interp/InterpStack.h
@@ -15,6 +15,7 @@
 
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
+#include "MemberPointer.h"
 #include "PrimType.h"
 #include <memory>
 #include <vector>
@@ -188,6 +189,8 @@ private:
       return PT_IntAP;
     else if constexpr (std::is_same_v<T, IntegralAP<false>>)
       return PT_IntAP;
+    else if constexpr (std::is_same_v<T, MemberPointer>)
+      return PT_MemberPtr;
 
     llvm_unreachable("unknown type push()'ed into InterpStack");
   }
diff --git a/clang/lib/AST/Interp/MemberPointer.cpp b/clang/lib/AST/Interp/MemberPointer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..96f63643e83c98f808346731dcd9e0d9610db4f5
--- /dev/null
+++ b/clang/lib/AST/Interp/MemberPointer.cpp
@@ -0,0 +1,76 @@
+//===------------------------- MemberPointer.cpp ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MemberPointer.h"
+#include "Context.h"
+#include "FunctionPointer.h"
+#include "Program.h"
+#include "Record.h"
+
+namespace clang {
+namespace interp {
+
+std::optional<Pointer> MemberPointer::toPointer(const Context &Ctx) const {
+  if (!Dcl || isa<FunctionDecl>(Dcl))
+    return Base;
+  const FieldDecl *FD = cast<FieldDecl>(Dcl);
+  assert(FD);
+
+  if (!Base.isBlockPointer())
+    return std::nullopt;
+
+  Pointer CastedBase =
+      (PtrOffset < 0 ? Base.atField(-PtrOffset) : Base.atFieldSub(PtrOffset));
+
+  const Record *BaseRecord = CastedBase.getRecord();
+  if (!BaseRecord)
+    return std::nullopt;
+
+  assert(BaseRecord);
+  if (FD->getParent() == BaseRecord->getDecl())
+    return CastedBase.atField(BaseRecord->getField(FD)->Offset);
+
+  const RecordDecl *FieldParent = FD->getParent();
+  const Record *FieldRecord = Ctx.getRecord(FieldParent);
+
+  unsigned Offset = 0;
+  Offset += FieldRecord->getField(FD)->Offset;
+  Offset += CastedBase.block()->getDescriptor()->getMetadataSize();
+
+  if (Offset > CastedBase.block()->getSize())
+    return std::nullopt;
+
+  if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl();
+      BaseDecl != FieldParent)
+    Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl);
+
+  if (Offset > CastedBase.block()->getSize())
+    return std::nullopt;
+
+  assert(Offset <= CastedBase.block()->getSize());
+  return Pointer(const_cast<Block *>(Base.block()), Offset, Offset);
+}
+
+FunctionPointer MemberPointer::toFunctionPointer(const Context &Ctx) const {
+  return FunctionPointer(Ctx.getProgram().getFunction(cast<FunctionDecl>(Dcl)));
+}
+
+APValue MemberPointer::toAPValue() const {
+  if (isZero())
+    return APValue(static_cast<ValueDecl *>(nullptr), /*IsDerivedMember=*/false,
+                   /*Path=*/{});
+
+  if (hasBase())
+    return Base.toAPValue();
+
+  return APValue(cast<ValueDecl>(getDecl()), /*IsDerivedMember=*/false,
+                 /*Path=*/{});
+}
+
+} // namespace interp
+} // namespace clang
diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/Interp/MemberPointer.h
new file mode 100644
index 0000000000000000000000000000000000000000..f56dc530431e46068ce72d2dea9870800da1b8c4
--- /dev/null
+++ b/clang/lib/AST/Interp/MemberPointer.h
@@ -0,0 +1,112 @@
+//===------------------------- MemberPointer.h ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_INTERP_MEMBER_POINTER_H
+#define LLVM_CLANG_AST_INTERP_MEMBER_POINTER_H
+
+#include "Pointer.h"
+#include <optional>
+
+namespace clang {
+class ASTContext;
+namespace interp {
+
+class Context;
+class FunctionPointer;
+
+class MemberPointer final {
+private:
+  Pointer Base;
+  const Decl *Dcl = nullptr;
+  int32_t PtrOffset = 0;
+
+  MemberPointer(Pointer Base, const Decl *Dcl, int32_t PtrOffset)
+      : Base(Base), Dcl(Dcl), PtrOffset(PtrOffset) {}
+
+public:
+  MemberPointer() = default;
+  MemberPointer(Pointer Base, const Decl *Dcl) : Base(Base), Dcl(Dcl) {}
+  MemberPointer(uint32_t Address, const Descriptor *D) {
+    // We only reach this for Address == 0, when creating a null member pointer.
+    assert(Address == 0);
+  }
+
+  MemberPointer(const Decl *D) : Dcl(D) {
+    assert((isa<FieldDecl, IndirectFieldDecl, CXXMethodDecl>(D)));
+  }
+
+  uint64_t getIntegerRepresentation() const {
+    assert(
+        false &&
+        "getIntegerRepresentation() shouldn't be reachable for MemberPointers");
+    return 17;
+  }
+
+  std::optional<Pointer> toPointer(const Context &Ctx) const;
+
+  FunctionPointer toFunctionPointer(const Context &Ctx) const;
+
+  Pointer getBase() const {
+    if (PtrOffset < 0)
+      return Base.atField(-PtrOffset);
+    return Base.atFieldSub(PtrOffset);
+  }
+  bool isMemberFunctionPointer() const {
+    return isa_and_nonnull<CXXMethodDecl>(Dcl);
+  }
+  const CXXMethodDecl *getMemberFunction() const {
+    return dyn_cast_if_present<CXXMethodDecl>(Dcl);
+  }
+  const FieldDecl *getField() const {
+    return dyn_cast_if_present<FieldDecl>(Dcl);
+  }
+
+  bool hasDecl() const { return Dcl; }
+  const Decl *getDecl() const { return Dcl; }
+
+  MemberPointer atInstanceBase(unsigned Offset) const {
+    if (Base.isZero())
+      return MemberPointer(Base, Dcl, Offset);
+    return MemberPointer(this->Base, Dcl, Offset + PtrOffset);
+  }
+
+  MemberPointer takeInstance(Pointer Instance) const {
+    assert(this->Base.isZero());
+    return MemberPointer(Instance, this->Dcl, this->PtrOffset);
+  }
+
+  APValue toAPValue() const;
+
+  bool isZero() const { return Base.isZero() && !Dcl; }
+  bool hasBase() const { return !Base.isZero(); }
+
+  void print(llvm::raw_ostream &OS) const {
+    OS << "MemberPtr(" << Base << " " << (const void *)Dcl << " + " << PtrOffset
+       << ")";
+  }
+
+  std::string toDiagnosticString(const ASTContext &Ctx) const {
+    return "FIXME";
+  }
+
+  ComparisonCategoryResult compare(const MemberPointer &RHS) const {
+    if (this->Dcl == RHS.Dcl)
+      return ComparisonCategoryResult::Equal;
+    return ComparisonCategoryResult::Unordered;
+  }
+};
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, MemberPointer FP) {
+  FP.print(OS);
+  return OS;
+}
+
+} // namespace interp
+} // namespace clang
+
+#endif
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index cfbd7f93c32de83552dc5973d3c9076345ac2cac..ac5426c87c2123ec6659c4e80dda8f3c6f81f251 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -30,6 +30,7 @@ def IntAPS : Type;
 def Float : Type;
 def Ptr : Type;
 def FnPtr : Type;
+def MemberPtr : Type;
 
 //===----------------------------------------------------------------------===//
 // Types transferred to the interpreter.
@@ -61,6 +62,7 @@ def ArgOffsetOfExpr : ArgType { let Name = "const OffsetOfExpr *"; }
 def ArgDeclRef : ArgType { let Name = "const DeclRefExpr *"; }
 def ArgDesc : ArgType { let Name = "const Descriptor *"; }
 def ArgCCI : ArgType { let Name = "const ComparisonCategoryInfo *"; }
+def ArgDecl : ArgType { let Name = "const Decl*"; }
 
 //===----------------------------------------------------------------------===//
 // Classes of types instructions operate on.
@@ -93,7 +95,7 @@ def AluTypeClass : TypeClass {
 }
 
 def PtrTypeClass : TypeClass {
-  let Types = [Ptr, FnPtr];
+  let Types = [Ptr, FnPtr, MemberPtr];
 }
 
 def BoolTypeClass : TypeClass {
@@ -208,7 +210,6 @@ def CallBI : Opcode {
 
 def CallPtr : Opcode {
   let Args = [ArgUint32, ArgCallExpr];
-  let Types = [];
 }
 
 def CallVar : Opcode {
@@ -280,54 +281,37 @@ def Null : Opcode {
 //===----------------------------------------------------------------------===//
 // Pointer generation
 //===----------------------------------------------------------------------===//
+class OffsetOpcode : Opcode {
+  let Args = [ArgUint32];
+}
 
 // [] -> [Pointer]
-def GetPtrLocal : Opcode {
-  // Offset of local.
-  let Args = [ArgUint32];
+def GetPtrLocal : OffsetOpcode {
   bit HasCustomEval = 1;
 }
 // [] -> [Pointer]
-def GetPtrParam : Opcode {
-  // Offset of parameter.
-  let Args = [ArgUint32];
-}
+def GetPtrParam : OffsetOpcode;
 // [] -> [Pointer]
-def GetPtrGlobal : Opcode {
-  // Index of global.
-  let Args = [ArgUint32];
-}
+def GetPtrGlobal : OffsetOpcode;
 // [Pointer] -> [Pointer]
-def GetPtrField : Opcode {
-  // Offset of field.
-  let Args = [ArgUint32];
-}
+def GetPtrField : OffsetOpcode;
+def GetPtrFieldPop : OffsetOpcode;
 // [Pointer] -> [Pointer]
-def GetPtrActiveField : Opcode {
-  // Offset of field.
-  let Args = [ArgUint32];
-}
+def GetPtrActiveField : OffsetOpcode;
 // [] -> [Pointer]
-def GetPtrActiveThisField : Opcode {
-  // Offset of field.
-  let Args = [ArgUint32];
-}
+def GetPtrActiveThisField : OffsetOpcode;
 // [] -> [Pointer]
-def GetPtrThisField : Opcode {
-  // Offset of field.
-  let Args = [ArgUint32];
-}
+def GetPtrThisField : OffsetOpcode;
 // [Pointer] -> [Pointer]
-def GetPtrBase : Opcode {
-  // Offset of field, which is a base.
-  let Args = [ArgUint32];
-}
+def GetPtrBase : OffsetOpcode;
 // [Pointer] -> [Pointer]
-def GetPtrBasePop : Opcode {
+def GetPtrBasePop : OffsetOpcode;
+def GetMemberPtrBasePop : Opcode {
   // Offset of field, which is a base.
-  let Args = [ArgUint32];
+  let Args = [ArgSint32];
 }
 
+
 def FinishInitPop : Opcode;
 def FinishInit    : Opcode;
 
@@ -681,6 +665,7 @@ def CastPointerIntegralAPS : Opcode {
   let HasGroup = 0;
   let Args = [ArgUint32];
 }
+def VoidPtrCast : Opcode;
 
 def DecayPtr : Opcode {
   let Types = [PtrTypeClass, PtrTypeClass];
@@ -751,6 +736,14 @@ def CheckNonNullArg : Opcode {
 
 def Memcpy : Opcode;
 
+def ToMemberPtr : Opcode;
+def CastMemberPtrPtr : Opcode;
+def GetMemberPtr : Opcode {
+  let Args = [ArgDecl];
+}
+def GetMemberPtrBase : Opcode;
+def GetMemberPtrDecl : Opcode;
+
 //===----------------------------------------------------------------------===//
 // Debugging.
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index 252f7ea46086f9c742fd980c2b4e6a774bb0fcc2..85857d4ee1c8897f9391f7ceb6d9a6e01ddd043a 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -13,6 +13,7 @@
 #include "Function.h"
 #include "Integral.h"
 #include "InterpBlock.h"
+#include "MemberPointer.h"
 #include "PrimType.h"
 #include "Record.h"
 
@@ -63,26 +64,27 @@ Pointer::~Pointer() {
 }
 
 void Pointer::operator=(const Pointer &P) {
-  if (!this->isIntegralPointer() || !P.isBlockPointer())
-    assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero()));
-
+  // If the current storage type is Block, we need to remove
+  // this pointer from the block.
   bool WasBlockPointer = isBlockPointer();
-  StorageKind = P.StorageKind;
   if (StorageKind == Storage::Block) {
     Block *Old = PointeeStorage.BS.Pointee;
-    if (WasBlockPointer && PointeeStorage.BS.Pointee)
+    if (WasBlockPointer && Old) {
       PointeeStorage.BS.Pointee->removePointer(this);
+      Old->cleanup();
+    }
+  }
 
-    Offset = P.Offset;
+  StorageKind = P.StorageKind;
+  Offset = P.Offset;
+
+  if (P.isBlockPointer()) {
     PointeeStorage.BS = P.PointeeStorage.BS;
+    PointeeStorage.BS.Pointee = P.PointeeStorage.BS.Pointee;
 
     if (PointeeStorage.BS.Pointee)
       PointeeStorage.BS.Pointee->addPointer(this);
-
-    if (WasBlockPointer && Old)
-      Old->cleanup();
-
-  } else if (StorageKind == Storage::Int) {
+  } else if (P.isIntegralPointer()) {
     PointeeStorage.Int = P.PointeeStorage.Int;
   } else {
     assert(false && "Unhandled storage kind");
@@ -90,26 +92,27 @@ void Pointer::operator=(const Pointer &P) {
 }
 
 void Pointer::operator=(Pointer &&P) {
-  if (!this->isIntegralPointer() || !P.isBlockPointer())
-    assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero()));
-
+  // If the current storage type is Block, we need to remove
+  // this pointer from the block.
   bool WasBlockPointer = isBlockPointer();
-  StorageKind = P.StorageKind;
   if (StorageKind == Storage::Block) {
     Block *Old = PointeeStorage.BS.Pointee;
-    if (WasBlockPointer && PointeeStorage.BS.Pointee)
+    if (WasBlockPointer && Old) {
       PointeeStorage.BS.Pointee->removePointer(this);
+      Old->cleanup();
+    }
+  }
 
-    Offset = P.Offset;
+  StorageKind = P.StorageKind;
+  Offset = P.Offset;
+
+  if (P.isBlockPointer()) {
     PointeeStorage.BS = P.PointeeStorage.BS;
+    PointeeStorage.BS.Pointee = P.PointeeStorage.BS.Pointee;
 
     if (PointeeStorage.BS.Pointee)
       PointeeStorage.BS.Pointee->addPointer(this);
-
-    if (WasBlockPointer && Old)
-      Old->cleanup();
-
-  } else if (StorageKind == Storage::Int) {
+  } else if (P.isIntegralPointer()) {
     PointeeStorage.Int = P.PointeeStorage.Int;
   } else {
     assert(false && "Unhandled storage kind");
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index 93ca754d04a644a119dc3893296dd668de3240a9..c6e4f4d0b4abd2cc5a006120041f57083d6a5414 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -620,6 +620,7 @@ public:
 private:
   friend class Block;
   friend class DeadBlock;
+  friend class MemberPointer;
   friend struct InitMap;
 
   Pointer(Block *Pointee, unsigned Base, uint64_t Offset);
diff --git a/clang/lib/AST/Interp/PrimType.cpp b/clang/lib/AST/Interp/PrimType.cpp
index 9b96dcfe6a272fc777bb7a2bb8f6c421d775d5bc..3054e67d5c49f36fde89cddce63dc8ff81b4280f 100644
--- a/clang/lib/AST/Interp/PrimType.cpp
+++ b/clang/lib/AST/Interp/PrimType.cpp
@@ -11,6 +11,7 @@
 #include "Floating.h"
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
+#include "MemberPointer.h"
 #include "Pointer.h"
 
 using namespace clang;
diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/Interp/PrimType.h
index 604fb5dfde1e416983fb7028bc3d3b7c71e23ca9..20fb5e81774d6ff47234a43e42f39c092d584bfa 100644
--- a/clang/lib/AST/Interp/PrimType.h
+++ b/clang/lib/AST/Interp/PrimType.h
@@ -25,6 +25,7 @@ class Pointer;
 class Boolean;
 class Floating;
 class FunctionPointer;
+class MemberPointer;
 template <bool Signed> class IntegralAP;
 template <unsigned Bits, bool Signed> class Integral;
 
@@ -44,10 +45,11 @@ enum PrimType : unsigned {
   PT_Float = 11,
   PT_Ptr = 12,
   PT_FnPtr = 13,
+  PT_MemberPtr = 14,
 };
 
 inline constexpr bool isPtrType(PrimType T) {
-  return T == PT_Ptr || T == PT_FnPtr;
+  return T == PT_Ptr || T == PT_FnPtr || T == PT_MemberPtr;
 }
 
 enum class CastKind : uint8_t {
@@ -91,6 +93,9 @@ template <> struct PrimConv<PT_Ptr> { using T = Pointer; };
 template <> struct PrimConv<PT_FnPtr> {
   using T = FunctionPointer;
 };
+template <> struct PrimConv<PT_MemberPtr> {
+  using T = MemberPointer;
+};
 
 /// Returns the size of a primitive type in bytes.
 size_t primSize(PrimType Type);
@@ -131,6 +136,7 @@ static inline bool aligned(const void *P) {
       TYPE_SWITCH_CASE(PT_Bool, B)                                             \
       TYPE_SWITCH_CASE(PT_Ptr, B)                                              \
       TYPE_SWITCH_CASE(PT_FnPtr, B)                                            \
+      TYPE_SWITCH_CASE(PT_MemberPtr, B)                                        \
     }                                                                          \
   } while (0)
 
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 403ce9a2215713a8849d27f786e3ecabf02582ce..95089a9b79e2671575bd41471a4814b9e105d6fc 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -104,7 +104,7 @@ OpenACCClause::child_range OpenACCClause::children() {
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
   case OpenACCClauseKind::CLAUSE_NAME:                                         \
     return cast<OpenACC##CLAUSE_NAME##Clause>(this)->children();
-#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME)                                  \
+#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED)                      \
   case OpenACCClauseKind::ALIAS_NAME:                                          \
     return cast<OpenACC##CLAUSE_NAME##Clause>(this)->children();
 
diff --git a/clang/lib/AST/ParentMap.cpp b/clang/lib/AST/ParentMap.cpp
index 534793b837bbb7aa5060a555c6a3d17f58325198..3d6a1cc84c7b10616ac4d87952a6157df4964f18 100644
--- a/clang/lib/AST/ParentMap.cpp
+++ b/clang/lib/AST/ParentMap.cpp
@@ -97,22 +97,6 @@ static void BuildParentMap(MapTy& M, Stmt* S,
       BuildParentMap(M, SubStmt, OVMode);
     }
     break;
-  case Stmt::CXXDefaultArgExprClass:
-    if (auto *Arg = dyn_cast<CXXDefaultArgExpr>(S)) {
-      if (Arg->hasRewrittenInit()) {
-        M[Arg->getExpr()] = S;
-        BuildParentMap(M, Arg->getExpr(), OVMode);
-      }
-    }
-    break;
-  case Stmt::CXXDefaultInitExprClass:
-    if (auto *Init = dyn_cast<CXXDefaultInitExpr>(S)) {
-      if (Init->hasRewrittenInit()) {
-        M[Init->getExpr()] = S;
-        BuildParentMap(M, Init->getExpr(), OVMode);
-      }
-    }
-    break;
   default:
     for (Stmt *SubStmt : S->children()) {
       if (SubStmt) {
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 8bacceea0f22b48622cdcfacc4a6556eb0fbc446..1076dcd40a694a570b9c69879257f051b50f788d 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -958,6 +958,9 @@ void TextNodeDumper::dumpTemplateArgument(const TemplateArgument &TA) {
   }
   OS << " '" << Str << "'";
 
+  if (!Context)
+    return;
+
   if (TemplateArgument CanonTA = Context->getCanonicalTemplateArgument(TA);
       !CanonTA.structurallyEquals(TA)) {
     llvm::SmallString<128> CanonStr;
@@ -1139,15 +1142,17 @@ void TextNodeDumper::dumpTemplateName(TemplateName TN, StringRef Label) {
       }
       OS << " '" << Str << "'";
 
-      if (TemplateName CanonTN = Context->getCanonicalTemplateName(TN);
-          CanonTN != TN) {
-        llvm::SmallString<128> CanonStr;
-        {
-          llvm::raw_svector_ostream SS(CanonStr);
-          CanonTN.print(SS, PrintPolicy);
+      if (Context) {
+        if (TemplateName CanonTN = Context->getCanonicalTemplateName(TN);
+            CanonTN != TN) {
+          llvm::SmallString<128> CanonStr;
+          {
+            llvm::raw_svector_ostream SS(CanonStr);
+            CanonTN.print(SS, PrintPolicy);
+          }
+          if (CanonStr != Str)
+            OS << ":'" << CanonStr << "'";
         }
-        if (CanonStr != Str)
-          OS << ":'" << CanonStr << "'";
       }
     }
     dumpBareTemplateName(TN);
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 2097b29b7e0b6d972ea562938ddc375c9af2a414..33acae2cbafacf62846613953e86f753eef70512 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2749,6 +2749,43 @@ bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
                                      /*IsCopyConstructible=*/false);
 }
 
+// FIXME: each call will trigger a full computation, cache the result.
+bool QualType::isBitwiseCloneableType(const ASTContext &Context) const {
+  auto CanonicalType = getCanonicalType();
+  if (CanonicalType.hasNonTrivialObjCLifetime())
+    return false;
+  if (CanonicalType->isArrayType())
+    return Context.getBaseElementType(CanonicalType)
+        .isBitwiseCloneableType(Context);
+
+  if (CanonicalType->isIncompleteType())
+    return false;
+  const auto *RD = CanonicalType->getAsRecordDecl(); // struct/union/class
+  if (!RD)
+    return true;
+
+  // Never allow memcpy when we're adding poisoned padding bits to the struct.
+  // Accessing these posioned bits will trigger false alarms on
+  // SanitizeAddressFieldPadding etc.
+  if (RD->mayInsertExtraPadding())
+    return false;
+
+  for (auto *const Field : RD->fields()) {
+    if (!Field->getType().isBitwiseCloneableType(Context))
+      return false;
+  }
+
+  if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+    for (auto Base : CXXRD->bases())
+      if (!Base.getType().isBitwiseCloneableType(Context))
+        return false;
+    for (auto VBase : CXXRD->vbases())
+      if (!VBase.getType().isBitwiseCloneableType(Context))
+        return false;
+  }
+  return true;
+}
+
 bool QualType::isTriviallyCopyConstructibleType(
     const ASTContext &Context) const {
   return isTriviallyCopyableTypeImpl(*this, Context,
@@ -4444,7 +4481,6 @@ static CachedProperties computeCachedProperties(const Type *T) {
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class,Base) case Type::Class:
 #include "clang/AST/TypeNodes.inc"
     // Treat instantiation-dependent types as external.
-    if (!T->isInstantiationDependentType()) T->dump();
     assert(T->isInstantiationDependentType());
     return CachedProperties(Linkage::External, false);
 
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index 02317257c2740523b8a3747464659de9842fe6b5..64e6155de090c5f3e0626968485ca104baa00f9e 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -556,10 +556,6 @@ public:
 
 private:
   // Visitors to walk an AST and construct the CFG.
-  CFGBlock *VisitCXXDefaultArgExpr(CXXDefaultArgExpr *Default,
-                                   AddStmtChoice asc);
-  CFGBlock *VisitCXXDefaultInitExpr(CXXDefaultInitExpr *Default,
-                                    AddStmtChoice asc);
   CFGBlock *VisitInitListExpr(InitListExpr *ILE, AddStmtChoice asc);
   CFGBlock *VisitAddrLabelExpr(AddrLabelExpr *A, AddStmtChoice asc);
   CFGBlock *VisitAttributedStmt(AttributedStmt *A, AddStmtChoice asc);
@@ -2258,10 +2254,16 @@ CFGBlock *CFGBuilder::Visit(Stmt * S, AddStmtChoice asc,
                                    asc, ExternallyDestructed);
 
     case Stmt::CXXDefaultArgExprClass:
-      return VisitCXXDefaultArgExpr(cast<CXXDefaultArgExpr>(S), asc);
-
     case Stmt::CXXDefaultInitExprClass:
-      return VisitCXXDefaultInitExpr(cast<CXXDefaultInitExpr>(S), asc);
+      // FIXME: The expression inside a CXXDefaultArgExpr is owned by the
+      // called function's declaration, not by the caller. If we simply add
+      // this expression to the CFG, we could end up with the same Expr
+      // appearing multiple times (PR13385).
+      //
+      // It's likewise possible for multiple CXXDefaultInitExprs for the same
+      // expression to be used in the same function (through aggregate
+      // initialization).
+      return VisitStmt(S, asc);
 
     case Stmt::CXXBindTemporaryExprClass:
       return VisitCXXBindTemporaryExpr(cast<CXXBindTemporaryExpr>(S), asc);
@@ -2431,40 +2433,6 @@ CFGBlock *CFGBuilder::VisitChildren(Stmt *S) {
   return B;
 }
 
-CFGBlock *CFGBuilder::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *Arg,
-                                             AddStmtChoice asc) {
-  if (Arg->hasRewrittenInit()) {
-    if (asc.alwaysAdd(*this, Arg)) {
-      autoCreateBlock();
-      appendStmt(Block, Arg);
-    }
-    return VisitStmt(Arg->getExpr(), asc);
-  }
-
-  // We can't add the default argument if it's not rewritten because the
-  // expression inside a CXXDefaultArgExpr is owned by the called function's
-  // declaration, not by the caller, we could end up with the same expression
-  // appearing multiple times.
-  return VisitStmt(Arg, asc);
-}
-
-CFGBlock *CFGBuilder::VisitCXXDefaultInitExpr(CXXDefaultInitExpr *Init,
-                                              AddStmtChoice asc) {
-  if (Init->hasRewrittenInit()) {
-    if (asc.alwaysAdd(*this, Init)) {
-      autoCreateBlock();
-      appendStmt(Block, Init);
-    }
-    return VisitStmt(Init->getExpr(), asc);
-  }
-
-  // We can't add the default initializer if it's not rewritten because multiple
-  // CXXDefaultInitExprs for the same sub-expression to be used in the same
-  // function (through aggregate initialization). we could end up with the same
-  // expression appearing multiple times.
-  return VisitStmt(Init, asc);
-}
-
 CFGBlock *CFGBuilder::VisitInitListExpr(InitListExpr *ILE, AddStmtChoice asc) {
   if (asc.alwaysAdd(*this, ILE)) {
     autoCreateBlock();
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index e2609b9573cca5e3e0b1202645cb474ef320850a..1d96a929f95d855cb87141723deb6f55a42c7fce 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -144,6 +144,7 @@ static const CudaArchToStringMap arch_names[] = {
     GFX(1103), // gfx1103
     GFX(1150), // gfx1150
     GFX(1151), // gfx1151
+    GFX(1152), // gfx1152
     {CudaArch::GFX12_GENERIC, "gfx12-generic", "compute_amdgcn"},
     GFX(1200), // gfx1200
     GFX(1201), // gfx1201
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index dc1792b3471e6c89be0d26644abad127da6150f9..29133f9ee8fcecceacbdf28d184946b061bb6515 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -673,8 +673,11 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple,
   }
   case llvm::Triple::spirv64: {
     if (os != llvm::Triple::UnknownOS ||
-        Triple.getEnvironment() != llvm::Triple::UnknownEnvironment)
+        Triple.getEnvironment() != llvm::Triple::UnknownEnvironment) {
+      if (os == llvm::Triple::OSType::AMDHSA)
+        return std::make_unique<SPIRV64AMDGCNTargetInfo>(Triple, Opts);
       return nullptr;
+    }
     return std::make_unique<SPIRV64TargetInfo>(Triple, Opts);
   }
   case llvm::Triple::wasm32:
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 0db16d4a8971abff969d2ce8770157c1b8c99241..14f4abec0b456daca2107a1476c5086d0bd0c412 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1050,57 +1050,18 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   return true;
 }
 
-bool AArch64TargetInfo::initFeatureMap(
-    llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
-    const std::vector<std::string> &FeaturesVec) const {
-  std::vector<std::string> UpdatedFeaturesVec;
-  // Parse the CPU and add any implied features.
-  std::optional<llvm::AArch64::CpuInfo> CpuInfo = llvm::AArch64::parseCpu(CPU);
-  if (CpuInfo) {
-    auto Exts = CpuInfo->getImpliedExtensions();
-    std::vector<StringRef> CPUFeats;
-    llvm::AArch64::getExtensionFeatures(Exts, CPUFeats);
-    for (auto F : CPUFeats) {
-      assert((F[0] == '+' || F[0] == '-') && "Expected +/- in target feature!");
-      UpdatedFeaturesVec.push_back(F.str());
-    }
-  }
-
-  // Process target and dependent features. This is done in two loops collecting
-  // them into UpdatedFeaturesVec: first to add dependent '+'features, second to
-  // add target '+/-'features that can later disable some of features added on
-  // the first loop. Function Multi Versioning features begin with '?'.
-  for (const auto &Feature : FeaturesVec)
-    if (((Feature[0] == '?' || Feature[0] == '+')) &&
-        AArch64TargetInfo::doesFeatureAffectCodeGen(Feature.substr(1))) {
-      StringRef DepFeatures =
-          AArch64TargetInfo::getFeatureDependencies(Feature.substr(1));
-      SmallVector<StringRef, 1> AttrFeatures;
-      DepFeatures.split(AttrFeatures, ",");
-      for (auto F : AttrFeatures)
-        UpdatedFeaturesVec.push_back(F.str());
-    }
-  for (const auto &Feature : FeaturesVec)
-    if (Feature[0] != '?') {
-      std::string UpdatedFeature = Feature;
-      if (Feature[0] == '+') {
-        std::optional<llvm::AArch64::ExtensionInfo> Extension =
-          llvm::AArch64::parseArchExtension(Feature.substr(1));
-        if (Extension)
-          UpdatedFeature = Extension->Feature.str();
-      }
-      UpdatedFeaturesVec.push_back(UpdatedFeature);
-    }
-
-  return TargetInfo::initFeatureMap(Features, Diags, CPU, UpdatedFeaturesVec);
-}
-
 // Parse AArch64 Target attributes, which are a comma separated list of:
 //  "arch=<arch>" - parsed to features as per -march=..
 //  "cpu=<cpu>" - parsed to features as per -mcpu=.., with CPU set to <cpu>
 //  "tune=<cpu>" - TuneCPU set to <cpu>
 //  "feature", "no-feature" - Add (or remove) feature.
 //  "+feature", "+nofeature" - Add (or remove) feature.
+//
+// A feature may correspond to an Extension (anything with a corresponding
+// AEK_), in which case an ExtensionSet is used to parse it and expand its
+// dependencies. Otherwise the feature is passed through (e.g. +v8.1a,
+// +outline-atomics, -fmv, etc). Features coming from the command line are
+// already parsed, therefore their dependencies do not need expansion.
 ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
   ParsedTargetAttr Ret;
   if (Features == "default")
@@ -1110,23 +1071,26 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
   bool FoundArch = false;
 
   auto SplitAndAddFeatures = [](StringRef FeatString,
-                                std::vector<std::string> &Features) {
+                                std::vector<std::string> &Features,
+                                llvm::AArch64::ExtensionSet &FeatureBits) {
     SmallVector<StringRef, 8> SplitFeatures;
     FeatString.split(SplitFeatures, StringRef("+"), -1, false);
     for (StringRef Feature : SplitFeatures) {
-      StringRef FeatureName = llvm::AArch64::getArchExtFeature(Feature);
-      if (!FeatureName.empty())
-        Features.push_back(FeatureName.str());
+      if (FeatureBits.parseModifier(Feature, /* AllowNoDashForm = */ true))
+        continue;
+      // Pass through features that are not extensions, e.g. +v8.1a,
+      // +outline-atomics, -fmv, etc.
+      if (Feature.starts_with("no"))
+        Features.push_back("-" + Feature.drop_front(2).str());
       else
-        // Pushing the original feature string to give a sema error later on
-        // when they get checked.
-        if (Feature.starts_with("no"))
-          Features.push_back("-" + Feature.drop_front(2).str());
-        else
-          Features.push_back("+" + Feature.str());
+        Features.push_back("+" + Feature.str());
     }
   };
 
+  llvm::AArch64::ExtensionSet FeatureBits;
+  // Reconstruct the bitset from the command line option features.
+  FeatureBits.reconstructFromParsedFeatures(getTargetOpts().FeaturesAsWritten);
+
   for (auto &Feature : AttrFeatures) {
     Feature = Feature.trim();
     if (Feature.starts_with("fpmath="))
@@ -1149,9 +1113,9 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
       // Ret.Features.
       if (!AI)
         continue;
-      Ret.Features.push_back(AI->ArchFeature.str());
+      FeatureBits.addArchDefaults(*AI);
       // Add any extra features, after the +
-      SplitAndAddFeatures(Split.second, Ret.Features);
+      SplitAndAddFeatures(Split.second, Ret.Features, FeatureBits);
     } else if (Feature.starts_with("cpu=")) {
       if (!Ret.CPU.empty())
         Ret.Duplicate = "cpu=";
@@ -1161,7 +1125,10 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
         std::pair<StringRef, StringRef> Split =
             Feature.split("=").second.trim().split("+");
         Ret.CPU = Split.first;
-        SplitAndAddFeatures(Split.second, Ret.Features);
+        if (auto CpuInfo = llvm::AArch64::parseCpu(Ret.CPU)) {
+          FeatureBits.addCPUDefaults(*CpuInfo);
+          SplitAndAddFeatures(Split.second, Ret.Features, FeatureBits);
+        }
       }
     } else if (Feature.starts_with("tune=")) {
       if (!Ret.Tune.empty())
@@ -1169,25 +1136,19 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
       else
         Ret.Tune = Feature.split("=").second.trim();
     } else if (Feature.starts_with("+")) {
-      SplitAndAddFeatures(Feature, Ret.Features);
-    } else if (Feature.starts_with("no-")) {
-      StringRef FeatureName =
-          llvm::AArch64::getArchExtFeature(Feature.split("-").second);
-      if (!FeatureName.empty())
-        Ret.Features.push_back("-" + FeatureName.drop_front(1).str());
-      else
-        Ret.Features.push_back("-" + Feature.split("-").second.str());
+      SplitAndAddFeatures(Feature, Ret.Features, FeatureBits);
     } else {
-      // Try parsing the string to the internal target feature name. If it is
-      // invalid, add the original string (which could already be an internal
-      // name). These should be checked later by isValidFeatureName.
-      StringRef FeatureName = llvm::AArch64::getArchExtFeature(Feature);
-      if (!FeatureName.empty())
-        Ret.Features.push_back(FeatureName.str());
+      if (FeatureBits.parseModifier(Feature, /* AllowNoDashForm = */ true))
+        continue;
+      // Pass through features that are not extensions, e.g. +v8.1a,
+      // +outline-atomics, -fmv, etc.
+      if (Feature.starts_with("no-"))
+        Ret.Features.push_back("-" + Feature.drop_front(3).str());
       else
         Ret.Features.push_back("+" + Feature.str());
     }
   }
+  FeatureBits.toLLVMFeatureList(Ret.Features);
   return Ret;
 }
 
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 12fb50286f7511a6c5bf461ba3cf041d5f8bda4b..696553ef8038a822fae018961d577dfca97ecdf0 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -107,10 +107,6 @@ public:
   unsigned multiVersionSortPriority(StringRef Name) const override;
   unsigned multiVersionFeatureCost() const override;
 
-  bool
-  initFeatureMap(llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags,
-                 StringRef CPU,
-                 const std::vector<std::string> &FeaturesVec) const override;
   bool useFP16ConversionIntrinsics() const override {
     return false;
   }
diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h
index 68572843f2d7486398361271abda589ab075ca69..5fc223483951e92a32f333a84f36e8430acadcb7 100644
--- a/clang/lib/Basic/Targets/LoongArch.h
+++ b/clang/lib/Basic/Targets/LoongArch.h
@@ -133,7 +133,7 @@ public:
     LongWidth = LongAlign = PointerWidth = PointerAlign = 64;
     IntMaxType = Int64Type = SignedLong;
     HasUnalignedAccess = true;
-    resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n64-S128");
+    resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
     // TODO: select appropriate ABI.
     setABI("lp64d");
   }
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index fc6ef1119e9cf788847f3a2117c90f73e6e68d21..ff7d2f1f92aa41156244694d3c8475308de3a7a0 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -228,6 +228,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
       case CudaArch::GFX1103:
       case CudaArch::GFX1150:
       case CudaArch::GFX1151:
+      case CudaArch::GFX1152:
       case CudaArch::GFX12_GENERIC:
       case CudaArch::GFX1200:
       case CudaArch::GFX1201:
diff --git a/clang/lib/Basic/Targets/SPIR.cpp b/clang/lib/Basic/Targets/SPIR.cpp
index dc920177d3a910723bb1a26b4b485686633e1e3f..040303983594f8b3c1d76e376ac4583ebd408b2e 100644
--- a/clang/lib/Basic/Targets/SPIR.cpp
+++ b/clang/lib/Basic/Targets/SPIR.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "SPIR.h"
+#include "AMDGPU.h"
 #include "Targets.h"
+#include "llvm/TargetParser/TargetParser.h"
 
 using namespace clang;
 using namespace clang::targets;
@@ -54,3 +56,76 @@ void SPIRV64TargetInfo::getTargetDefines(const LangOptions &Opts,
   BaseSPIRVTargetInfo::getTargetDefines(Opts, Builder);
   DefineStd(Builder, "SPIRV64", Opts);
 }
+
+static const AMDGPUTargetInfo AMDGPUTI(llvm::Triple("amdgcn-amd-amdhsa"), {});
+
+ArrayRef<const char *> SPIRV64AMDGCNTargetInfo::getGCCRegNames() const {
+  return AMDGPUTI.getGCCRegNames();
+}
+
+bool SPIRV64AMDGCNTargetInfo::initFeatureMap(
+    llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef,
+    const std::vector<std::string> &FeatureVec) const {
+  llvm::AMDGPU::fillAMDGPUFeatureMap({}, getTriple(), Features);
+
+  return TargetInfo::initFeatureMap(Features, Diags, {}, FeatureVec);
+}
+
+bool SPIRV64AMDGCNTargetInfo::validateAsmConstraint(
+    const char *&Name, TargetInfo::ConstraintInfo &Info) const {
+  return AMDGPUTI.validateAsmConstraint(Name, Info);
+}
+
+std::string
+SPIRV64AMDGCNTargetInfo::convertConstraint(const char *&Constraint) const {
+  return AMDGPUTI.convertConstraint(Constraint);
+}
+
+ArrayRef<Builtin::Info> SPIRV64AMDGCNTargetInfo::getTargetBuiltins() const {
+  return AMDGPUTI.getTargetBuiltins();
+}
+
+void SPIRV64AMDGCNTargetInfo::getTargetDefines(const LangOptions &Opts,
+                                               MacroBuilder &Builder) const {
+  BaseSPIRVTargetInfo::getTargetDefines(Opts, Builder);
+  DefineStd(Builder, "SPIRV64", Opts);
+
+  Builder.defineMacro("__AMD__");
+  Builder.defineMacro("__AMDGPU__");
+  Builder.defineMacro("__AMDGCN__");
+}
+
+void SPIRV64AMDGCNTargetInfo::setAuxTarget(const TargetInfo *Aux) {
+  assert(Aux && "Cannot invoke setAuxTarget without a valid auxiliary target!");
+
+  // This is a 1:1 copy of AMDGPUTargetInfo::setAuxTarget()
+  assert(HalfFormat == Aux->HalfFormat);
+  assert(FloatFormat == Aux->FloatFormat);
+  assert(DoubleFormat == Aux->DoubleFormat);
+
+  // On x86_64 long double is 80-bit extended precision format, which is
+  // not supported by AMDGPU. 128-bit floating point format is also not
+  // supported by AMDGPU. Therefore keep its own format for these two types.
+  auto SaveLongDoubleFormat = LongDoubleFormat;
+  auto SaveFloat128Format = Float128Format;
+  auto SaveLongDoubleWidth = LongDoubleWidth;
+  auto SaveLongDoubleAlign = LongDoubleAlign;
+  copyAuxTarget(Aux);
+  LongDoubleFormat = SaveLongDoubleFormat;
+  Float128Format = SaveFloat128Format;
+  LongDoubleWidth = SaveLongDoubleWidth;
+  LongDoubleAlign = SaveLongDoubleAlign;
+  // For certain builtin types support on the host target, claim they are
+  // supported to pass the compilation of the host code during the device-side
+  // compilation.
+  // FIXME: As the side effect, we also accept `__float128` uses in the device
+  // code. To reject these builtin types supported in the host target but not in
+  // the device target, one approach would support `device_builtin` attribute
+  // so that we could tell the device builtin types from the host ones. This
+  // also solves the different representations of the same builtin type, such
+  // as `size_t` in the MSVC environment.
+  if (Aux->hasFloat128Type()) {
+    HasFloat128 = true;
+    Float128Format = DoubleFormat;
+  }
+}
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 44265445ff004b0858266d2c3678636ed5514ac1..37cf9d7921bac52a8177d71ec018977d1cb11ede 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -364,6 +364,57 @@ public:
                         MacroBuilder &Builder) const override;
 };
 
+class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
+    : public BaseSPIRVTargetInfo {
+public:
+  SPIRV64AMDGCNTargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts)
+      : BaseSPIRVTargetInfo(Triple, Opts) {
+    assert(Triple.getArch() == llvm::Triple::spirv64 &&
+           "Invalid architecture for 64-bit AMDGCN SPIR-V.");
+    assert(Triple.getVendor() == llvm::Triple::VendorType::AMD &&
+           "64-bit AMDGCN SPIR-V target must use AMD vendor");
+    assert(getTriple().getOS() == llvm::Triple::OSType::AMDHSA &&
+           "64-bit AMDGCN SPIR-V target must use AMDHSA OS");
+    assert(getTriple().getEnvironment() == llvm::Triple::UnknownEnvironment &&
+           "64-bit SPIR-V target must use unknown environment type");
+    PointerWidth = PointerAlign = 64;
+    SizeType = TargetInfo::UnsignedLong;
+    PtrDiffType = IntPtrType = TargetInfo::SignedLong;
+
+    resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-"
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0");
+
+    BFloat16Width = BFloat16Align = 16;
+    BFloat16Format = &llvm::APFloat::BFloat();
+
+    HasLegalHalfType = true;
+    HasFloat16 = true;
+    HalfArgsAndReturns = true;
+  }
+
+  bool hasBFloat16Type() const override { return true; }
+
+  ArrayRef<const char *> getGCCRegNames() const override;
+
+  bool initFeatureMap(llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags,
+                      StringRef,
+                      const std::vector<std::string> &) const override;
+
+  bool validateAsmConstraint(const char *&Name,
+                             TargetInfo::ConstraintInfo &Info) const override;
+
+  std::string convertConstraint(const char *&Constraint) const override;
+
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+
+  void getTargetDefines(const LangOptions &Opts,
+                        MacroBuilder &Builder) const override;
+
+  void setAuxTarget(const TargetInfo *Aux) override;
+
+  bool hasInt128Type() const override { return TargetInfo::hasInt128Type(); }
+};
+
 } // namespace targets
 } // namespace clang
 #endif // LLVM_CLANG_LIB_BASIC_TARGETS_SPIR_H
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 37d0c478e03308cf1713f5ffef3a9f3b1e08582e..c16b69ba875679ff75b102de391207718d422214 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -6012,6 +6012,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
   if (!Prefix.empty()) {
     IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
+    if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
+        getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
+      IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin("amdgcn", Name);
     // NOTE we don't need to perform a compatibility flag check here since the
     // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
     // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
@@ -6182,6 +6185,10 @@ static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
     return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
+  case llvm::Triple::spirv64:
+    if (CGF->getTarget().getTriple().getOS() != llvm::Triple::OSType::AMDHSA)
+      return nullptr;
+    return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
   default:
     return nullptr;
   }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 97449a5e51e736c80fa085fe0da20900acdabe17..65d82285b907b521a371322153ad5f009864dc0c 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1917,6 +1917,7 @@ static void getTrivialDefaultFunctionAttributes(
     case CodeGenOptions::FramePointerKind::None:
       // This is the default behavior.
       break;
+    case CodeGenOptions::FramePointerKind::Reserved:
     case CodeGenOptions::FramePointerKind::NonLeaf:
     case CodeGenOptions::FramePointerKind::All:
       FuncAttrs.addAttribute("frame-pointer",
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 5f6f911c7a6d698d44324e94a85c0259f6ed7084..11e2d549d8a4505e9c92d0aeed0bb016284e2483 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -58,7 +58,16 @@ using namespace clang::CodeGen;
 
 static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) {
   auto TI = Ctx.getTypeInfo(Ty);
-  return TI.isAlignRequired() ? TI.Align : 0;
+  if (TI.isAlignRequired())
+    return TI.Align;
+
+  // MaxFieldAlignmentAttr is the attribute added to types
+  // declared after #pragma pack(n).
+  if (auto *Decl = Ty->getAsRecordDecl())
+    if (Decl->hasAttr<MaxFieldAlignmentAttr>())
+      return TI.Align;
+
+  return 0;
 }
 
 static uint32_t getTypeAlignIfRequired(QualType Ty, const ASTContext &Ctx) {
@@ -5757,28 +5766,16 @@ void CGDebugInfo::EmitPseudoVariable(CGBuilderTy &Builder,
   // it is loaded upon use, so we identify such pattern here.
   if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Value)) {
     llvm::Value *Var = Load->getPointerOperand();
-    if (llvm::Metadata *MDValue = llvm::ValueAsMetadata::getIfExists(Var)) {
-      if (llvm::Value *DbgValue = llvm::MetadataAsValue::getIfExists(
-              CGM.getLLVMContext(), MDValue)) {
-        for (llvm::User *U : DbgValue->users()) {
-          if (llvm::CallInst *DbgDeclare = dyn_cast<llvm::CallInst>(U)) {
-            if (DbgDeclare->getCalledFunction()->getIntrinsicID() ==
-                    llvm::Intrinsic::dbg_declare &&
-                DbgDeclare->getArgOperand(0) == DbgValue) {
-              // There can be implicit type cast applied on a variable if it is
-              // an opaque ptr, in this case its debug info may not match the
-              // actual type of object being used as in the next instruction, so
-              // we will need to emit a pseudo variable for type-casted value.
-              llvm::DILocalVariable *MDNode = cast<llvm::DILocalVariable>(
-                  cast<llvm::MetadataAsValue>(DbgDeclare->getOperand(1))
-                      ->getMetadata());
-              if (MDNode->getType() == Type)
-                return;
-            }
-          }
-        }
-      }
-    }
+    // There can be implicit type cast applied on a variable if it is an opaque
+    // ptr, in this case its debug info may not match the actual type of object
+    // being used as in the next instruction, so we will need to emit a pseudo
+    // variable for type-casted value.
+    auto DeclareTypeMatches = [&](auto *DbgDeclare) {
+      return DbgDeclare->getVariable()->getType() == Type;
+    };
+    if (any_of(llvm::findDbgDeclares(Var), DeclareTypeMatches) ||
+        any_of(llvm::findDVRDeclares(Var), DeclareTypeMatches))
+      return;
   }
 
   // Find the correct location to insert a sequence of instructions to
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 5b2039af6128bf0de90e324608566b63d3ea8b65..b2a5ceeeae08ba16d88fee4087469a9f394c5b59 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -513,15 +513,6 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
 
   QualType elementType =
       CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();
-
-  // DestPtr is an array*.  Construct an elementType* by drilling
-  // down a level.
-  llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
-  llvm::Value *indices[] = { zero, zero };
-  llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(),
-                                                 DestPtr.emitRawPointer(CGF),
-                                                 indices, "arrayinit.begin");
-
   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
   CharUnits elementAlign =
     DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
@@ -562,6 +553,7 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
   Address endOfInit = Address::invalid();
   CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
 
+  llvm::Value *begin = DestPtr.emitRawPointer(CGF);
   if (dtorKind) {
     CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
     // In principle we could tell the cleanup where we are more
@@ -585,19 +577,13 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
 
   llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);
 
-  // The 'current element to initialize'.  The invariants on this
-  // variable are complicated.  Essentially, after each iteration of
-  // the loop, it points to the last initialized element, except
-  // that it points to the beginning of the array before any
-  // elements have been initialized.
-  llvm::Value *element = begin;
-
   // Emit the explicit initializers.
   for (uint64_t i = 0; i != NumInitElements; ++i) {
-    // Advance to the next element.
+    llvm::Value *element = begin;
     if (i > 0) {
-      element = Builder.CreateInBoundsGEP(
-          llvmElementType, element, one, "arrayinit.element");
+      element = Builder.CreateInBoundsGEP(llvmElementType, begin,
+                                          llvm::ConstantInt::get(CGF.SizeTy, i),
+                                          "arrayinit.element");
 
       // Tell the cleanup that it needs to destroy up to this
       // element.  TODO: some of these stores can be trivially
@@ -624,9 +610,12 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
     //   do { *array++ = filler; } while (array != end);
 
     // Advance to the start of the rest of the array.
+    llvm::Value *element = begin;
     if (NumInitElements) {
       element = Builder.CreateInBoundsGEP(
-          llvmElementType, element, one, "arrayinit.start");
+          llvmElementType, element,
+          llvm::ConstantInt::get(CGF.SizeTy, NumInitElements),
+          "arrayinit.start");
       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
     }
 
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 5e6a3dd4878f46550216ed46f9cc94e59d18d522..55ba21ae2ba69ba1c0997cca3e3f03833b894896 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -313,7 +313,7 @@ void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
   assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
   const StringRef ShaderAttrKindStr = "hlsl.shader";
   Fn->addFnAttr(ShaderAttrKindStr,
-                ShaderAttr->ConvertShaderTypeToStr(ShaderAttr->getType()));
+                llvm::Triple::getEnvironmentTypeName(ShaderAttr->getType()));
   if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
     const StringRef NumThreadsKindStr = "hlsl.numthreads";
     std::string NumThreadsStr =
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 191bd757831fc75f161e0d0c8071b6033065c7c0..6e9a1bacd9bf5f4a3f5a840a5c25f9ccd75e0405 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -3537,6 +3537,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(
       case CudaArch::GFX1103:
       case CudaArch::GFX1150:
       case CudaArch::GFX1151:
+      case CudaArch::GFX1152:
       case CudaArch::GFX12_GENERIC:
       case CudaArch::GFX1200:
       case CudaArch::GFX1201:
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 41ac511c52f51b887c916dd8cdfb8d1e7611e775..39222c0e65353b20b571643a92f0656da64a2291 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -414,7 +414,8 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
     CGM.ErrorUnsupported(S, "OpenMP dispatch directive");
     break;
   case Stmt::OMPScopeDirectiveClass:
-    llvm_unreachable("scope not supported with FE outlining");
+    CGM.ErrorUnsupported(S, "scope with FE outlining");
+    break;
   case Stmt::OMPMaskedDirectiveClass:
     EmitOMPMaskedDirective(cast<OMPMaskedDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index be7bf0b72dc0c1563e8d15acafa11f8281b9b0d3..75b14490903890b0edcb9f037716d59782bd8df4 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1328,6 +1328,9 @@ void CodeGenModule::Release() {
   case CodeGenOptions::FramePointerKind::None:
     // 0 ("none") is the default.
     break;
+  case CodeGenOptions::FramePointerKind::Reserved:
+    getModule().setFramePointer(llvm::FramePointerKind::Reserved);
+    break;
   case CodeGenOptions::FramePointerKind::NonLeaf:
     getModule().setFramePointer(llvm::FramePointerKind::NonLeaf);
     break;
diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp
index d1ff8b4b62f1570ed545946bf0e198cf1661346a..057f6ef40c5133d5589d09ecb4ed75d64662e3eb 100644
--- a/clang/lib/CodeGen/Targets/AMDGPU.cpp
+++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp
@@ -120,7 +120,11 @@ void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
 Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
                                  QualType Ty) const {
-  llvm_unreachable("AMDGPU does not support varargs");
+  const bool IsIndirect = false;
+  const bool AllowHigherAlign = false;
+  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
+                          getContext().getTypeInfoInChars(Ty),
+                          CharUnits::fromQuantity(4), AllowHigherAlign);
 }
 
 ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const {
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 0e86bc07e0ea2c5d5265bc0a1422f535e70d0449..40ab2e91125d1e531254a7d6d0a9d6db9e14afc0 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -104,7 +104,8 @@ ToolChain::ToolChain(const Driver &D, const llvm::Triple &T,
 }
 
 llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
-ToolChain::executeToolChainProgram(StringRef Executable) const {
+ToolChain::executeToolChainProgram(StringRef Executable,
+                                   unsigned SecondsToWait) const {
   llvm::SmallString<64> OutputFile;
   llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile);
   llvm::FileRemover OutputRemover(OutputFile.c_str());
@@ -115,9 +116,8 @@ ToolChain::executeToolChainProgram(StringRef Executable) const {
   };
 
   std::string ErrorMessage;
-  if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects,
-                                /* SecondsToWait */ 0,
-                                /*MemoryLimit*/ 0, &ErrorMessage))
+  if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects, SecondsToWait,
+                                /*MemoryLimit=*/0, &ErrorMessage))
     return llvm::createStringError(std::error_code(),
                                    Executable + ": " + ErrorMessage);
 
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 9ffea57b005de0085bee4a75bf36793897f66ca8..20f879e2f75cb809a3114cbe03dc84f7a6f369e0 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -645,7 +645,11 @@ void amdgpu::getAMDGPUTargetFeatures(const Driver &D,
                                      std::vector<StringRef> &Features) {
   // Add target ID features to -target-feature options. No diagnostics should
   // be emitted here since invalid target ID is diagnosed at other places.
-  StringRef TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ);
+  StringRef TargetID;
+  if (Args.hasArg(options::OPT_mcpu_EQ))
+    TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ);
+  else if (Args.hasArg(options::OPT_march_EQ))
+    TargetID = Args.getLastArgValue(options::OPT_march_EQ);
   if (!TargetID.empty()) {
     llvm::StringMap<bool> FeatureMap;
     auto OptionalGpuArch = parseTargetID(Triple, TargetID, &FeatureMap);
@@ -877,7 +881,7 @@ AMDGPUToolChain::getSystemGPUArchs(const ArgList &Args) const {
   else
     Program = GetProgramPath("amdgpu-arch");
 
-  auto StdoutOrErr = executeToolChainProgram(Program);
+  auto StdoutOrErr = executeToolChainProgram(Program, /*SecondsToWait=*/10);
   if (!StdoutOrErr)
     return StdoutOrErr.takeError();
 
diff --git a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
index cca18431ff773240e73aa35a7d49a8e0820b8daf..d17ecb15c82088e4a0afcc6e9d8d57fbef2e7e43 100644
--- a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
@@ -44,14 +44,9 @@ void AMDGPUOpenMPToolChain::addClangTargetOptions(
     Action::OffloadKind DeviceOffloadingKind) const {
   HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
 
-  StringRef GPUArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
-  assert(!GPUArch.empty() && "Must have an explicit GPU arch.");
-
   assert(DeviceOffloadingKind == Action::OFK_OpenMP &&
          "Only OpenMP offloading kinds are supported.");
 
-  CC1Args.push_back("-target-cpu");
-  CC1Args.push_back(DriverArgs.MakeArgStringRef(GPUArch));
   CC1Args.push_back("-fcuda-is-device");
 
   if (DriverArgs.hasArg(options::OPT_nogpulib))
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index a68368c4758651ab9fb60863df294b63285d5582..8ae22cc37a136867a941a7dab1f312f88f544b8b 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -799,8 +799,6 @@ fp16_fml_fallthrough:
     StringRef FrameChainOption = A->getValue();
     if (FrameChainOption.starts_with("aapcs"))
       Features.push_back("+aapcs-frame-chain");
-    if (FrameChainOption == "aapcs+leaf")
-      Features.push_back("+aapcs-frame-chain-leaf");
   }
 
   // CMSE: Check for target 8M (for -mcmse to be applicable) is performed later.
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 2e2bce8494672fe0e29086fbf768de1d3c256b31..26789b0ba6e098966013048e73ab32b2c7b86035 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -67,11 +67,6 @@ static void getRISCFeaturesFromMcpu(const Driver &D, const Arg *A,
       D.Diag(clang::diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Mcpu;
   }
-
-  if (llvm::RISCV::hasFastUnalignedAccess(Mcpu)) {
-    Features.push_back("+unaligned-scalar-mem");
-    Features.push_back("+unaligned-vector-mem");
-  }
 }
 
 void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
@@ -82,6 +77,8 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   if (!getArchFeatures(D, MArch, Features, Args))
     return;
 
+  bool CPUFastUnaligned = false;
+
   // If users give march and mcpu, get std extension feature from MArch
   // and other features (ex. mirco architecture feature) from mcpu
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
@@ -90,6 +87,9 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       CPU = llvm::sys::getHostCPUName();
 
     getRISCFeaturesFromMcpu(D, A, Triple, CPU, Features);
+
+    if (llvm::RISCV::hasFastUnalignedAccess(CPU))
+      CPUFastUnaligned = true;
   }
 
   // Handle features corresponding to "-ffixed-X" options
@@ -169,18 +169,23 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
     Features.push_back("-relax");
   }
 
-  // Android requires fast unaligned access on RISCV64.
-  if (Triple.isAndroid()) {
+  // If -mstrict-align or -mno-strict-align is passed, use it. Otherwise, the
+  // unaligned-*-mem is enabled if the CPU supports it or the target is
+  // Android.
+  if (const Arg *A = Args.getLastArg(options::OPT_mno_strict_align,
+                                     options::OPT_mstrict_align)) {
+    if (A->getOption().matches(options::OPT_mno_strict_align)) {
+      Features.push_back("+unaligned-scalar-mem");
+      Features.push_back("+unaligned-vector-mem");
+    } else {
+      Features.push_back("-unaligned-scalar-mem");
+      Features.push_back("-unaligned-vector-mem");
+    }
+  } else if (CPUFastUnaligned || Triple.isAndroid()) {
     Features.push_back("+unaligned-scalar-mem");
     Features.push_back("+unaligned-vector-mem");
   }
 
-  // -mstrict-align is default, unless -mno-strict-align is specified.
-  AddTargetFeature(Args, Features, options::OPT_mno_strict_align,
-                   options::OPT_mstrict_align, "unaligned-scalar-mem");
-  AddTargetFeature(Args, Features, options::OPT_mno_strict_align,
-                   options::OPT_mstrict_align, "unaligned-vector-mem");
-
   // Now add any that the user explicitly requested on the command line,
   // which may override the defaults.
   handleTargetFeaturesGroup(D, Triple, Args, Features,
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 221c4815792405a4d02fd02d41245dc1f36db5bc..dd365e62e084e68468ddd42bece4a6b0eb12bc39 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -429,6 +429,7 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   ArgStringList CmdArgs;
 
   auto &TC = static_cast<const toolchains::BareMetal &>(getToolChain());
+  const Driver &D = getToolChain().getDriver();
   const llvm::Triple::ArchType Arch = TC.getArch();
   const llvm::Triple &Triple = getToolChain().getEffectiveTriple();
 
@@ -466,6 +467,19 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     TC.AddLinkRuntimeLib(Args, CmdArgs);
   }
 
+  if (D.isUsingLTO()) {
+    assert(!Inputs.empty() && "Must have at least one input.");
+    // Find the first filename InputInfo object.
+    auto Input = llvm::find_if(
+        Inputs, [](const InputInfo &II) -> bool { return II.isFilename(); });
+    if (Input == Inputs.end())
+      // For a very rare case, all of the inputs to the linker are
+      // InputArg. If that happens, just use the first InputInfo.
+      Input = Inputs.begin();
+
+    addLTOOptions(TC, Args, CmdArgs, Output, *Input,
+                  D.getLTOMode() == LTOK_Thin);
+  }
   if (TC.getTriple().isRISCV())
     CmdArgs.push_back("-X");
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 4e1c52462e5842c8cce009195ab37e9fd9ff1e71..b8d8ff3db5d1fd5e082bcda4733d3a2352782fa1 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5678,6 +5678,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   case CodeGenOptions::FramePointerKind::None:
     FPKeepKindStr = "-mframe-pointer=none";
     break;
+  case CodeGenOptions::FramePointerKind::Reserved:
+    FPKeepKindStr = "-mframe-pointer=reserved";
+    break;
   case CodeGenOptions::FramePointerKind::NonLeaf:
     FPKeepKindStr = "-mframe-pointer=non-leaf";
     break;
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 71e993119436aed2cdb3c5caada8fa249b13aba8..2a4c1369f5a734917cadada1c92d5de497d43d6d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -164,6 +164,14 @@ static bool useFramePointerForTargetByDefault(const llvm::opt::ArgList &Args,
   return true;
 }
 
+static bool useLeafFramePointerForTargetByDefault(const llvm::Triple &Triple) {
+  if (Triple.isAArch64() || Triple.isPS() || Triple.isVE() ||
+      (Triple.isAndroid() && Triple.isRISCV64()))
+    return false;
+
+  return true;
+}
+
 static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) {
   switch (Triple.getArch()) {
   default:
@@ -176,38 +184,91 @@ static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) {
   }
 }
 
+// True if a target-specific option requires the frame chain to be preserved,
+// even if new frame records are not created.
+static bool mustMaintainValidFrameChain(const llvm::opt::ArgList &Args,
+                                        const llvm::Triple &Triple) {
+  if (Triple.isARM() || Triple.isThumb()) {
+    // For 32-bit Arm, the -mframe-chain=aapcs and -mframe-chain=aapcs+leaf
+    // options require the frame pointer register to be reserved (or point to a
+    // new AAPCS-compilant frame record), even with	-fno-omit-frame-pointer.
+    if (Arg *A = Args.getLastArg(options::OPT_mframe_chain)) {
+      StringRef V = A->getValue();
+      return V != "none";
+    }
+    return false;
+  }
+  return false;
+}
+
+// True if a target-specific option causes -fno-omit-frame-pointer to also
+// cause frame records to be created in leaf functions.
+static bool framePointerImpliesLeafFramePointer(const llvm::opt::ArgList &Args,
+                                                const llvm::Triple &Triple) {
+  if (Triple.isARM() || Triple.isThumb()) {
+    // For 32-bit Arm, the -mframe-chain=aapcs+leaf option causes the
+    // -fno-omit-frame-pointer optiion to imply -mno-omit-leaf-frame-pointer,
+    // but does not by itself imply either option.
+    if (Arg *A = Args.getLastArg(options::OPT_mframe_chain)) {
+      StringRef V = A->getValue();
+      return V == "aapcs+leaf";
+    }
+    return false;
+  }
+  return false;
+}
+
 clang::CodeGenOptions::FramePointerKind
 getFramePointerKind(const llvm::opt::ArgList &Args,
                     const llvm::Triple &Triple) {
-  // We have 4 states:
+  // There are three things to consider here:
+  // * Should a frame record be created for non-leaf functions?
+  // * Should a frame record be created for leaf functions?
+  // * Is the frame pointer register reserved, i.e. must it always point to
+  //   either a new, valid frame record or be un-modified?
   //
-  //  00) leaf retained, non-leaf retained
-  //  01) leaf retained, non-leaf omitted (this is invalid)
-  //  10) leaf omitted, non-leaf retained
-  //      (what -momit-leaf-frame-pointer was designed for)
-  //  11) leaf omitted, non-leaf omitted
+  //  Not all combinations of these are valid:
+  //  * It's not useful to have leaf frame records without non-leaf ones.
+  //  * It's not useful to have frame records without reserving the frame
+  //    pointer.
   //
-  //  "omit" options taking precedence over "no-omit" options is the only way
-  //  to make 3 valid states representable
-  llvm::opt::Arg *A =
-      Args.getLastArg(clang::driver::options::OPT_fomit_frame_pointer,
-                      clang::driver::options::OPT_fno_omit_frame_pointer);
-
-  bool OmitFP = A && A->getOption().matches(
-                         clang::driver::options::OPT_fomit_frame_pointer);
-  bool NoOmitFP = A && A->getOption().matches(
-                           clang::driver::options::OPT_fno_omit_frame_pointer);
-  bool OmitLeafFP =
-      Args.hasFlag(clang::driver::options::OPT_momit_leaf_frame_pointer,
-                   clang::driver::options::OPT_mno_omit_leaf_frame_pointer,
-                   Triple.isAArch64() || Triple.isPS() || Triple.isVE() ||
-                       (Triple.isAndroid() && Triple.isRISCV64()));
-  if (NoOmitFP || mustUseNonLeafFramePointerForTarget(Triple) ||
-      (!OmitFP && useFramePointerForTargetByDefault(Args, Triple))) {
-    if (OmitLeafFP)
-      return clang::CodeGenOptions::FramePointerKind::NonLeaf;
-    return clang::CodeGenOptions::FramePointerKind::All;
-  }
+  // | Non-leaf | Leaf | Reserved |
+  // | N        | N    | N        | FramePointerKind::None
+  // | N        | N    | Y        | FramePointerKind::Reserved
+  // | N        | Y    | N        | Invalid
+  // | N        | Y    | Y        | Invalid
+  // | Y        | N    | N        | Invalid
+  // | Y        | N    | Y        | FramePointerKind::NonLeaf
+  // | Y        | Y    | N        | Invalid
+  // | Y        | Y    | Y        | FramePointerKind::All
+  //
+  // The FramePointerKind::Reserved case is currently only reachable for Arm,
+  // which has the -mframe-chain= option which can (in combination with
+  // -fno-omit-frame-pointer) specify that the frame chain must be valid,
+  // without requiring new frame records to be created.
+
+  bool DefaultFP = useFramePointerForTargetByDefault(Args, Triple);
+  bool EnableFP =
+      mustUseNonLeafFramePointerForTarget(Triple) ||
+      Args.hasFlag(clang::driver::options::OPT_fno_omit_frame_pointer,
+                   clang::driver::options::OPT_fomit_frame_pointer, DefaultFP);
+
+  bool DefaultLeafFP =
+      useLeafFramePointerForTargetByDefault(Triple) ||
+      (EnableFP && framePointerImpliesLeafFramePointer(Args, Triple));
+  bool EnableLeafFP = Args.hasFlag(
+      clang::driver::options::OPT_mno_omit_leaf_frame_pointer,
+      clang::driver::options::OPT_momit_leaf_frame_pointer, DefaultLeafFP);
+
+  bool FPRegReserved = EnableFP || mustMaintainValidFrameChain(Args, Triple);
+
+  if (EnableFP) {
+    if (EnableLeafFP)
+      return clang::CodeGenOptions::FramePointerKind::All;
+    return clang::CodeGenOptions::FramePointerKind::NonLeaf;
+  }
+  if (FPRegReserved)
+    return clang::CodeGenOptions::FramePointerKind::Reserved;
   return clang::CodeGenOptions::FramePointerKind::None;
 }
 
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index bbc8be91fd70b701cb00f1f8bd61ec22cb149a9f..2dfc7457b0ac7c1475d9e7b1c9ad2169ea8c55dd 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -826,7 +826,7 @@ NVPTXToolChain::getSystemGPUArchs(const ArgList &Args) const {
   else
     Program = GetProgramPath("nvptx-arch");
 
-  auto StdoutOrErr = executeToolChainProgram(Program);
+  auto StdoutOrErr = executeToolChainProgram(Program, /*SecondsToWait=*/10);
   if (!StdoutOrErr)
     return StdoutOrErr.takeError();
 
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 9609a1dc65c095a79bf567dabb050ec223395bad..42b45dba2bd31116d8b8a6434f845755950c13b7 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -802,6 +802,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   case CodeGenOptions::FramePointerKind::None:
     FPKeepKindStr = "-mframe-pointer=none";
     break;
+   case CodeGenOptions::FramePointerKind::Reserved:
+    FPKeepKindStr = "-mframe-pointer=reserved";
+    break;
   case CodeGenOptions::FramePointerKind::NonLeaf:
     FPKeepKindStr = "-mframe-pointer=non-leaf";
     break;
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 08e711cafae280c761c35751d7ce0e8d495fe6f2..6e56ee5b573f66af612d0ebcfe9444abdb04bd3f 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -31,7 +31,6 @@
 using namespace clang;
 using namespace clang::extractapi;
 using namespace llvm;
-using namespace llvm::json;
 
 namespace {
 
@@ -1036,9 +1035,9 @@ void SymbolGraphSerializer::serializeGraphToStream(
     ExtendedModule &&EM) {
   Object Root = serializeGraph(ModuleName, std::move(EM));
   if (Options.Compact)
-    OS << formatv("{0}", Value(std::move(Root))) << "\n";
+    OS << formatv("{0}", json::Value(std::move(Root))) << "\n";
   else
-    OS << formatv("{0:2}", Value(std::move(Root))) << "\n";
+    OS << formatv("{0:2}", json::Value(std::move(Root))) << "\n";
 }
 
 void SymbolGraphSerializer::serializeMainSymbolGraph(
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 6b9fbfe0ebf53fad85a71e9688f2354a0545d20e..be684ac71cd614f7d03121d8d4ae4927dc140f0f 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1712,7 +1712,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
         (!Previous || Previous->isNot(tok::kw_return) ||
          (Style.Language != FormatStyle::LK_Java && PrecedenceLevel > 0)) &&
         (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign ||
-         PrecedenceLevel != prec::Comma || Current.NestingLevel == 0) &&
+         PrecedenceLevel > prec::Comma || Current.NestingLevel == 0) &&
         (!Style.isTableGen() ||
          (Previous && Previous->isOneOf(TT_TableGenDAGArgListComma,
                                         TT_TableGenDAGArgListCommaToBreak)))) {
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 26c0aa36bdcb64f9cfc98efd5a8cb139df179607..1fe3b61a5a81f2deae3c648525efaebc822770a0 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -1358,6 +1358,8 @@ private:
           Line.First->startsSequence(tok::kw_export, Keywords.kw_module) ||
           Line.First->startsSequence(tok::kw_export, Keywords.kw_import)) {
         Tok->setType(TT_ModulePartitionColon);
+      } else if (Line.First->is(tok::kw_asm)) {
+        Tok->setType(TT_InlineASMColon);
       } else if (Contexts.back().ColonIsDictLiteral || Style.isProto()) {
         Tok->setType(TT_DictLiteral);
         if (Style.Language == FormatStyle::LK_TextProto) {
@@ -1425,13 +1427,6 @@ private:
         // This handles a special macro in ObjC code where selectors including
         // the colon are passed as macro arguments.
         Tok->setType(TT_ObjCMethodExpr);
-      } else if (Contexts.back().ContextKind == tok::l_paren &&
-                 !Line.InPragmaDirective) {
-        if (Style.isTableGen() && Contexts.back().IsTableGenDAGArg) {
-          Tok->setType(TT_TableGenDAGArgListColon);
-          break;
-        }
-        Tok->setType(TT_InlineASMColon);
       }
       break;
     case tok::pipe:
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index d6061c2666c2ad87f606eb39768c580822d75823..eb96b54ec4c962481bd498fc7d0504f3cef2d520 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -1181,10 +1181,10 @@ void UnwrappedLineParser::parsePPDefine() {
   Line->InMacroBody = true;
 
   if (Style.SkipMacroDefinitionBody) {
-    do {
+    while (!eof()) {
       FormatTok->Finalized = true;
-      nextToken();
-    } while (!eof());
+      FormatTok = Tokens->getNextToken();
+    }
     addUnwrappedLine();
     return;
   }
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 1812b85860f64cf835d77efef2ad8d01c3acf2e2..4f064321997a2268047b7634e09a803ce0ab8fb0 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -1169,8 +1169,8 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() {
   llvm::SmallVector<dependency_directives_scan::Token, 16> Tokens;
   llvm::SmallVector<dependency_directives_scan::Directive, 32> Directives;
   if (scanSourceForDependencyDirectives(
-          FromFile.getBuffer(), Tokens, Directives, CI.getLangOpts(),
-          &CI.getDiagnostics(), SM.getLocForStartOfFile(SM.getMainFileID()))) {
+          FromFile.getBuffer(), Tokens, Directives, &CI.getDiagnostics(),
+          SM.getLocForStartOfFile(SM.getMainFileID()))) {
     assert(CI.getDiagnostics().hasErrorOccurred() &&
            "no errors reported for failure");
 
diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp
index 5bc8385d874a16869edd75fdd22088237a834671..a8d0294fb6151bd057d8dd228fba6eaa208757ac 100644
--- a/clang/lib/Interpreter/IncrementalParser.cpp
+++ b/clang/lib/Interpreter/IncrementalParser.cpp
@@ -413,7 +413,8 @@ void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
     if (!ND)
       continue;
     // Check if we need to clean up the IdResolver chain.
-    if (ND->getDeclName().getFETokenInfo())
+    if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
+        !D->getLangOpts().CPlusPlus)
       getCI()->getSema().IdResolver.RemoveDecl(ND);
   }
 }
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index 683f87e8c8c79e36ed90da00151fc400708f3e87..7a95278914276af695d7fe234b29da2fae599a3a 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -42,6 +42,9 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
+
+#include <cstdarg>
+
 using namespace clang;
 
 // FIXME: Figure out how to unify with namespace init_convenience from
@@ -270,14 +273,10 @@ Interpreter::~Interpreter() {
 // can't find the precise resource directory in unittests so we have to hard
 // code them.
 const char *const Runtimes = R"(
+    #define __CLANG_REPL__ 1
 #ifdef __cplusplus
+    #define EXTERN_C extern "C"
     void *__clang_Interpreter_SetValueWithAlloc(void*, void*, void*);
-    void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*);
-    void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, void*);
-    void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, float);
-    void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, double);
-    void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, long double);
-    void __clang_Interpreter_SetValueNoAlloc(void*,void*,void*,unsigned long long);
     struct __clang_Interpreter_NewTag{} __ci_newtag;
     void* operator new(__SIZE_TYPE__, void* __p, __clang_Interpreter_NewTag) noexcept;
     template <class T, class = T (*)() /*disable for arrays*/>
@@ -289,7 +288,11 @@ const char *const Runtimes = R"(
     void __clang_Interpreter_SetValueCopyArr(const T (*Src)[N], void* Placement, unsigned long Size) {
       __clang_Interpreter_SetValueCopyArr(Src[0], Placement, Size);
     }
+#else
+    #define EXTERN_C extern
 #endif // __cplusplus
+
+  EXTERN_C void __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ...);
 )";
 
 llvm::Expected<std::unique_ptr<Interpreter>>
@@ -588,15 +591,17 @@ std::unique_ptr<RuntimeInterfaceBuilder> Interpreter::FindRuntimeInterface() {
   if (!LookupInterface(ValuePrintingInfo[NoAlloc],
                        MagicRuntimeInterface[NoAlloc]))
     return nullptr;
-  if (!LookupInterface(ValuePrintingInfo[WithAlloc],
-                       MagicRuntimeInterface[WithAlloc]))
-    return nullptr;
-  if (!LookupInterface(ValuePrintingInfo[CopyArray],
-                       MagicRuntimeInterface[CopyArray]))
-    return nullptr;
-  if (!LookupInterface(ValuePrintingInfo[NewTag],
-                       MagicRuntimeInterface[NewTag]))
-    return nullptr;
+  if (Ctx.getLangOpts().CPlusPlus) {
+    if (!LookupInterface(ValuePrintingInfo[WithAlloc],
+                         MagicRuntimeInterface[WithAlloc]))
+      return nullptr;
+    if (!LookupInterface(ValuePrintingInfo[CopyArray],
+                         MagicRuntimeInterface[CopyArray]))
+      return nullptr;
+    if (!LookupInterface(ValuePrintingInfo[NewTag],
+                         MagicRuntimeInterface[NewTag]))
+      return nullptr;
+  }
 
   return createInProcessRuntimeInterfaceBuilder(*this, Ctx, S);
 }
@@ -855,69 +860,81 @@ __clang_Interpreter_SetValueWithAlloc(void *This, void *OutVal,
   return VRef.getPtr();
 }
 
-// Pointers, lvalue struct that can take as a reference.
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
-                                    void *Val) {
+extern "C" void REPL_EXTERNAL_VISIBILITY __clang_Interpreter_SetValueNoAlloc(
+    void *This, void *OutVal, void *OpaqueType, ...) {
   Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-  VRef.setPtr(Val);
-}
+  Interpreter *I = static_cast<Interpreter *>(This);
+  VRef = Value(I, OpaqueType);
+  if (VRef.isVoid())
+    return;
 
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal,
-                                    void *OpaqueType) {
-  Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-}
+  va_list args;
+  va_start(args, /*last named param*/ OpaqueType);
 
-static void SetValueDataBasedOnQualType(Value &V, unsigned long long Data) {
-  QualType QT = V.getType();
-  if (const auto *ET = QT->getAs<EnumType>())
-    QT = ET->getDecl()->getIntegerType();
-
-  switch (QT->castAs<BuiltinType>()->getKind()) {
-  default:
-    llvm_unreachable("unknown type kind!");
-#define X(type, name)                                                          \
-  case BuiltinType::name:                                                      \
-    V.set##name(Data);                                                         \
-    break;
-    REPL_BUILTIN_TYPES
-#undef X
+  QualType QT = VRef.getType();
+  if (VRef.getKind() == Value::K_PtrOrObj) {
+    VRef.setPtr(va_arg(args, void *));
+  } else {
+    if (const auto *ET = QT->getAs<EnumType>())
+      QT = ET->getDecl()->getIntegerType();
+    switch (QT->castAs<BuiltinType>()->getKind()) {
+    default:
+      llvm_unreachable("unknown type kind!");
+      break;
+      // Types shorter than int are resolved as int, else va_arg has UB.
+    case BuiltinType::Bool:
+      VRef.setBool(va_arg(args, int));
+      break;
+    case BuiltinType::Char_S:
+      VRef.setChar_S(va_arg(args, int));
+      break;
+    case BuiltinType::SChar:
+      VRef.setSChar(va_arg(args, int));
+      break;
+    case BuiltinType::Char_U:
+      VRef.setChar_U(va_arg(args, unsigned));
+      break;
+    case BuiltinType::UChar:
+      VRef.setUChar(va_arg(args, unsigned));
+      break;
+    case BuiltinType::Short:
+      VRef.setShort(va_arg(args, int));
+      break;
+    case BuiltinType::UShort:
+      VRef.setUShort(va_arg(args, unsigned));
+      break;
+    case BuiltinType::Int:
+      VRef.setInt(va_arg(args, int));
+      break;
+    case BuiltinType::UInt:
+      VRef.setUInt(va_arg(args, unsigned));
+      break;
+    case BuiltinType::Long:
+      VRef.setLong(va_arg(args, long));
+      break;
+    case BuiltinType::ULong:
+      VRef.setULong(va_arg(args, unsigned long));
+      break;
+    case BuiltinType::LongLong:
+      VRef.setLongLong(va_arg(args, long long));
+      break;
+    case BuiltinType::ULongLong:
+      VRef.setULongLong(va_arg(args, unsigned long long));
+      break;
+      // Types shorter than double are resolved as double, else va_arg has UB.
+    case BuiltinType::Float:
+      VRef.setFloat(va_arg(args, double));
+      break;
+    case BuiltinType::Double:
+      VRef.setDouble(va_arg(args, double));
+      break;
+    case BuiltinType::LongDouble:
+      VRef.setLongDouble(va_arg(args, long double));
+      break;
+      // See REPL_BUILTIN_TYPES.
+    }
   }
-}
-
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
-                                    unsigned long long Val) {
-  Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-  SetValueDataBasedOnQualType(VRef, Val);
-}
-
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
-                                    float Val) {
-  Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-  VRef.setFloat(Val);
-}
-
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
-                                    double Val) {
-  Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-  VRef.setDouble(Val);
-}
-
-REPL_EXTERNAL_VISIBILITY void
-__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
-                                    long double Val) {
-  Value &VRef = *(Value *)OutVal;
-  VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
-  VRef.setLongDouble(Val);
+  va_end(args);
 }
 
 // A trampoline to work around the fact that operator placement new cannot
diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index fda54d314eef6c0d0981e615d98dbccc6246c124..0971daa1f36663f0bb17177e1b65f84dfddfc786 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -62,17 +62,14 @@ struct DirectiveWithTokens {
 struct Scanner {
   Scanner(StringRef Input,
           SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-          DiagnosticsEngine *Diags, SourceLocation InputSourceLoc,
-          const LangOptions &LangOpts)
+          DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
       : Input(Input), Tokens(Tokens), Diags(Diags),
-        InputSourceLoc(InputSourceLoc),
-        LangOpts(getLangOptsForDepScanning(LangOpts)),
-        TheLexer(InputSourceLoc, this->LangOpts, Input.begin(), Input.begin(),
+        InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
+        TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
                  Input.end()) {}
 
-  static LangOptions
-  getLangOptsForDepScanning(const LangOptions &invocationLangOpts) {
-    LangOptions LangOpts(invocationLangOpts);
+  static LangOptions getLangOptsForDepScanning() {
+    LangOptions LangOpts;
     // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
     LangOpts.ObjC = true;
     LangOpts.LineComment = true;
@@ -703,7 +700,7 @@ bool Scanner::lex_Pragma(const char *&First, const char *const End) {
   SmallVector<dependency_directives_scan::Token> DiscardTokens;
   const char *Begin = Buffer.c_str();
   Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
-                        InputSourceLoc, LangOptions()};
+                        InputSourceLoc};
 
   PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
   if (PragmaScanner.lexPragma(Begin, Buffer.end()))
@@ -953,10 +950,9 @@ bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
 
 bool clang::scanSourceForDependencyDirectives(
     StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-    SmallVectorImpl<Directive> &Directives, const LangOptions &LangOpts,
-    DiagnosticsEngine *Diags, SourceLocation InputSourceLoc) {
-  return Scanner(Input, Tokens, Diags, InputSourceLoc, LangOpts)
-      .scan(Directives);
+    SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
+    SourceLocation InputSourceLoc) {
+  return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
 }
 
 void clang::printDependencyDirectivesAsSource(
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index c25203243ee49bccf6f78f68eb5a02f242a0722e..16a5b7483ec1c3b6bee991a014c7f4725784e781 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -571,11 +571,8 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) {
   }
 
   Token *CurTok = nullptr;
-  // If the semicolon is missing at the end of REPL input, consider if
-  // we want to do value printing. Note this is only enabled in C++ mode
-  // since part of the implementation requires C++ language features.
   // Note we shouldn't eat the token since the callback needs it.
-  if (Tok.is(tok::annot_repl_input_end) && Actions.getLangOpts().CPlusPlus)
+  if (Tok.is(tok::annot_repl_input_end))
     CurTok = &Tok;
   else
     // Otherwise, eat the semicolon.
diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp
index c08073e80ff3dcecafc777942dcbfa05cde2621a..5bc7e79a681869c32c38f7d54064393d896fe311 100644
--- a/clang/lib/Sema/Scope.cpp
+++ b/clang/lib/Sema/Scope.cpp
@@ -228,7 +228,11 @@ void Scope::dumpImpl(raw_ostream &OS) const {
       {CompoundStmtScope, "CompoundStmtScope"},
       {ClassInheritanceScope, "ClassInheritanceScope"},
       {CatchScope, "CatchScope"},
+      {ConditionVarScope, "ConditionVarScope"},
+      {OpenMPOrderClauseScope, "OpenMPOrderClauseScope"},
+      {LambdaScope, "LambdaScope"},
       {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
+      {TypeAliasScope, "TypeAliasScope"},
       {FriendScope, "FriendScope"},
   };
 
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index c446cc1d042a4a60b0c68ec5212228fb49c180e8..d11bc9eec3301943bd3a431e372d554c8f10ba3e 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -31,9 +31,9 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID,
     constexpr const int SizeIdx = 2;
     llvm::APSInt Size;
     Expr *ArgExpr = TheCall->getArg(SizeIdx);
-    ExprResult R = SemaRef.VerifyIntegerConstantExpression(ArgExpr, &Size);
-    if (R.isInvalid())
-      return true;
+    [[maybe_unused]] ExprResult R =
+        SemaRef.VerifyIntegerConstantExpression(ArgExpr, &Size);
+    assert(!R.isInvalid());
     switch (Size.getSExtValue()) {
     case 1:
     case 2:
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index 330cd602297d46d7914180b4789e6a33887c6a79..3e5f90b450367b5cc8cc48592bd976a8064f2b7c 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -12,6 +12,7 @@
 
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/IdentifierTable.h"
@@ -46,6 +47,10 @@ static const AvailabilityAttr *getAttrForPlatform(ASTContext &Context,
   // Check each AvailabilityAttr to find the one for this platform.
   // For multiple attributes with the same platform try to find one for this
   // environment.
+  // The attribute is always on the FunctionDecl, not on the
+  // FunctionTemplateDecl.
+  if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(D))
+    D = FTD->getTemplatedDecl();
   for (const auto *A : D->attrs()) {
     if (const auto *Avail = dyn_cast<AvailabilityAttr>(A)) {
       // FIXME: this is copied from CheckAvailability. We should try to
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a6734ef8c30aa8b6c35b29d8239089ea32d09e7d..4b9b735f1cfb43a9735117a7dcfb18d06ef6e7ce 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2288,7 +2288,8 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
     // Partial translation units that are created in incremental processing must
     // not clean up the IdResolver because PTUs should take into account the
     // declarations that came from previous PTUs.
-    if (!PP.isIncrementalProcessingEnabled() || getLangOpts().ObjC)
+    if (!PP.isIncrementalProcessingEnabled() || getLangOpts().ObjC ||
+        getLangOpts().CPlusPlus)
       IdResolver.RemoveDecl(D);
 
     // Warn on it if we are shadowing a declaration.
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 3c28da1b077cd5f6dab90ae998b9f78fedde432a..37f0df2a6a27d5ff65bdc1e077ac6f33a986c881 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -1806,6 +1806,7 @@ static unsigned getRecordDiagFromTagKind(TagTypeKind Tag) {
 static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
                                        Stmt *Body,
                                        Sema::CheckConstexprKind Kind);
+static bool CheckConstexprMissingReturn(Sema &SemaRef, const FunctionDecl *Dcl);
 
 // Check whether a function declaration satisfies the requirements of a
 // constexpr function definition or a constexpr constructor definition. If so,
@@ -2411,20 +2412,9 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
     }
   } else {
     if (ReturnStmts.empty()) {
-      // C++1y doesn't require constexpr functions to contain a 'return'
-      // statement. We still do, unless the return type might be void, because
-      // otherwise if there's no return statement, the function cannot
-      // be used in a core constant expression.
-      bool OK = SemaRef.getLangOpts().CPlusPlus14 &&
-                (Dcl->getReturnType()->isVoidType() ||
-                 Dcl->getReturnType()->isDependentType());
       switch (Kind) {
       case Sema::CheckConstexprKind::Diagnose:
-        SemaRef.Diag(Dcl->getLocation(),
-                     OK ? diag::warn_cxx11_compat_constexpr_body_no_return
-                        : diag::err_constexpr_body_no_return)
-            << Dcl->isConsteval();
-        if (!OK)
+        if (!CheckConstexprMissingReturn(SemaRef, Dcl))
           return false;
         break;
 
@@ -2494,6 +2484,28 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
   return true;
 }
 
+static bool CheckConstexprMissingReturn(Sema &SemaRef,
+                                        const FunctionDecl *Dcl) {
+  bool IsVoidOrDependentType = Dcl->getReturnType()->isVoidType() ||
+                               Dcl->getReturnType()->isDependentType();
+  // Skip emitting a missing return error diagnostic for non-void functions
+  // since C++23 no longer mandates constexpr functions to yield constant
+  // expressions.
+  if (SemaRef.getLangOpts().CPlusPlus23 && !IsVoidOrDependentType)
+    return true;
+
+  // C++14 doesn't require constexpr functions to contain a 'return'
+  // statement. We still do, unless the return type might be void, because
+  // otherwise if there's no return statement, the function cannot
+  // be used in a core constant expression.
+  bool OK = SemaRef.getLangOpts().CPlusPlus14 && IsVoidOrDependentType;
+  SemaRef.Diag(Dcl->getLocation(),
+               OK ? diag::warn_cxx11_compat_constexpr_body_no_return
+                  : diag::err_constexpr_body_no_return)
+      << Dcl->isConsteval();
+  return OK;
+}
+
 bool Sema::CheckImmediateEscalatingFunctionDefinition(
     FunctionDecl *FD, const sema::FunctionScopeInfo *FSI) {
   if (!getLangOpts().CPlusPlus20 || !FD->isImmediateEscalating())
@@ -13079,7 +13091,10 @@ NamedDecl *Sema::BuildUsingDeclaration(
   // A using-declaration shall not name a namespace.
   if (R.getAsSingle<NamespaceDecl>()) {
     Diag(IdentLoc, diag::err_using_decl_can_not_refer_to_namespace)
-      << SS.getRange();
+        << SS.getRange();
+    // Suggest using 'using namespace ...' instead.
+    Diag(SS.getBeginLoc(), diag::note_namespace_using_decl)
+        << FixItHint::CreateInsertion(SS.getBeginLoc(), "namespace ");
     return BuildInvalid();
   }
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index fb5ca199b3fc62d95fadb4ee39020a8c587fcda3..76145f291887cbed0c499cb3ccc15cbfd6fa76d3 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5572,9 +5572,10 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
         Res = Immediate.TransformInitializer(Param->getInit(),
                                              /*NotCopy=*/false);
       });
-      if (Res.isUsable())
-        Res = ConvertParamDefaultArgument(Param, Res.get(),
-                                          Res.get()->getBeginLoc());
+      if (Res.isInvalid())
+        return ExprError();
+      Res = ConvertParamDefaultArgument(Param, Res.get(),
+                                        Res.get()->getBeginLoc());
       if (Res.isInvalid())
         return ExprError();
       Init = Res.get();
@@ -5608,10 +5609,9 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
     InitializationContext.emplace(Loc, Field, CurContext);
 
   Expr *Init = nullptr;
-  bool HasRewrittenInit = false;
 
   bool NestedDefaultChecking = isCheckingDefaultArgumentOrInitializer();
-  bool InLifetimeExtendingContext = isInLifetimeExtendingContext();
+
   EnterExpressionEvaluationContext EvalContext(
       *this, ExpressionEvaluationContext::PotentiallyEvaluated, Field);
 
@@ -5646,36 +5646,19 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   ImmediateCallVisitor V(getASTContext());
   if (!NestedDefaultChecking)
     V.TraverseDecl(Field);
-
-  // CWG1815
-  // Support lifetime extension of temporary created by aggregate
-  // initialization using a default member initializer. We should always rebuild
-  // the initializer if it contains any temporaries (if the initializer
-  // expression is an ExprWithCleanups). Then make sure the normal lifetime
-  // extension code recurses into the default initializer and does lifetime
-  // extension when warranted.
-  bool ContainsAnyTemporaries =
-      isa_and_present<ExprWithCleanups>(Field->getInClassInitializer());
-  if (V.HasImmediateCalls || InLifetimeExtendingContext ||
-      ContainsAnyTemporaries) {
-    HasRewrittenInit = true;
+  if (V.HasImmediateCalls) {
     ExprEvalContexts.back().DelayedDefaultInitializationContext = {Loc, Field,
                                                                    CurContext};
     ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer =
         NestedDefaultChecking;
-    // Pass down lifetime extending flag, and collect temporaries in
-    // CreateMaterializeTemporaryExpr when we rewrite the call argument.
-    keepInLifetimeExtendingContext();
+
     EnsureImmediateInvocationInDefaultArgs Immediate(*this);
     ExprResult Res;
-
-    // Rebuild CXXDefaultInitExpr might cause diagnostics.
-    SFINAETrap Trap(*this);
     runWithSufficientStackSpace(Loc, [&] {
       Res = Immediate.TransformInitializer(Field->getInClassInitializer(),
                                            /*CXXDirectInit=*/false);
     });
-    if (Res.isUsable())
+    if (!Res.isInvalid())
       Res = ConvertMemberDefaultInitExpression(Field, Res.get(), Loc);
     if (Res.isInvalid()) {
       Field->setInvalidDecl();
@@ -5702,7 +5685,7 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
 
     return CXXDefaultInitExpr::Create(Context, InitializationContext->Loc,
                                       Field, InitializationContext->Context,
-                                      HasRewrittenInit ? Init : nullptr);
+                                      Init);
   }
 
   // DR1351:
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 4487c618862c5e9efe37385dbad71eaa028c6b5d..cf461a68d55263eb1d444bf4e873f879777e280f 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1555,6 +1555,9 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
                                 bool ListInitialization) {
   QualType Ty = TInfo->getType();
   SourceLocation TyBeginLoc = TInfo->getTypeLoc().getBeginLoc();
+
+  assert((!ListInitialization || Exprs.size() == 1) &&
+         "List initialization must have exactly one expression.");
   SourceRange FullRange = SourceRange(TyBeginLoc, RParenOrBraceLoc);
 
   InitializedEntity Entity =
@@ -5126,6 +5129,7 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT,
   case UTT_IsStandardLayout:
   case UTT_IsPOD:
   case UTT_IsLiteral:
+  case UTT_IsBitwiseCloneable:
   // By analogy, is_trivially_relocatable and is_trivially_equality_comparable
   // impose the same constraints.
   case UTT_IsTriviallyRelocatable:
@@ -5619,6 +5623,8 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
     return C.hasUniqueObjectRepresentations(T);
   case UTT_IsTriviallyRelocatable:
     return T.isTriviallyRelocatableType(C);
+  case UTT_IsBitwiseCloneable:
+    return T.isBitwiseCloneableType(C);
   case UTT_IsReferenceable:
     return T.isReferenceable();
   case UTT_CanPassInRegs:
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 0a2face7afe65a7048d1509d573f11ca9acbdc01..144cdcc0d98ef14d3413e8755f7a120b3d95989b 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -146,7 +146,7 @@ HLSLNumThreadsAttr *SemaHLSL::mergeNumThreadsAttr(Decl *D,
 
 HLSLShaderAttr *
 SemaHLSL::mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
-                          HLSLShaderAttr::ShaderType ShaderType) {
+                          llvm::Triple::EnvironmentType ShaderType) {
   if (HLSLShaderAttr *NT = D->getAttr<HLSLShaderAttr>()) {
     if (NT->getType() != ShaderType) {
       Diag(NT->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
@@ -184,13 +184,12 @@ void SemaHLSL::ActOnTopLevelFunction(FunctionDecl *FD) {
   if (FD->getName() != TargetInfo.getTargetOpts().HLSLEntry)
     return;
 
-  StringRef Env = TargetInfo.getTriple().getEnvironmentName();
-  HLSLShaderAttr::ShaderType ShaderType;
-  if (HLSLShaderAttr::ConvertStrToShaderType(Env, ShaderType)) {
+  llvm::Triple::EnvironmentType Env = TargetInfo.getTriple().getEnvironment();
+  if (HLSLShaderAttr::isValidShaderType(Env) && Env != llvm::Triple::Library) {
     if (const auto *Shader = FD->getAttr<HLSLShaderAttr>()) {
       // The entry point is already annotated - check that it matches the
       // triple.
-      if (Shader->getType() != ShaderType) {
+      if (Shader->getType() != Env) {
         Diag(Shader->getLocation(), diag::err_hlsl_entry_shader_attr_mismatch)
             << Shader;
         FD->setInvalidDecl();
@@ -198,11 +197,11 @@ void SemaHLSL::ActOnTopLevelFunction(FunctionDecl *FD) {
     } else {
       // Implicitly add the shader attribute if the entry function isn't
       // explicitly annotated.
-      FD->addAttr(HLSLShaderAttr::CreateImplicit(getASTContext(), ShaderType,
+      FD->addAttr(HLSLShaderAttr::CreateImplicit(getASTContext(), Env,
                                                  FD->getBeginLoc()));
     }
   } else {
-    switch (TargetInfo.getTriple().getEnvironment()) {
+    switch (Env) {
     case llvm::Triple::UnknownEnvironment:
     case llvm::Triple::Library:
       break;
@@ -215,38 +214,40 @@ void SemaHLSL::ActOnTopLevelFunction(FunctionDecl *FD) {
 void SemaHLSL::CheckEntryPoint(FunctionDecl *FD) {
   const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
   assert(ShaderAttr && "Entry point has no shader attribute");
-  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
+  llvm::Triple::EnvironmentType ST = ShaderAttr->getType();
 
   switch (ST) {
-  case HLSLShaderAttr::Pixel:
-  case HLSLShaderAttr::Vertex:
-  case HLSLShaderAttr::Geometry:
-  case HLSLShaderAttr::Hull:
-  case HLSLShaderAttr::Domain:
-  case HLSLShaderAttr::RayGeneration:
-  case HLSLShaderAttr::Intersection:
-  case HLSLShaderAttr::AnyHit:
-  case HLSLShaderAttr::ClosestHit:
-  case HLSLShaderAttr::Miss:
-  case HLSLShaderAttr::Callable:
+  case llvm::Triple::Pixel:
+  case llvm::Triple::Vertex:
+  case llvm::Triple::Geometry:
+  case llvm::Triple::Hull:
+  case llvm::Triple::Domain:
+  case llvm::Triple::RayGeneration:
+  case llvm::Triple::Intersection:
+  case llvm::Triple::AnyHit:
+  case llvm::Triple::ClosestHit:
+  case llvm::Triple::Miss:
+  case llvm::Triple::Callable:
     if (const auto *NT = FD->getAttr<HLSLNumThreadsAttr>()) {
       DiagnoseAttrStageMismatch(NT, ST,
-                                {HLSLShaderAttr::Compute,
-                                 HLSLShaderAttr::Amplification,
-                                 HLSLShaderAttr::Mesh});
+                                {llvm::Triple::Compute,
+                                 llvm::Triple::Amplification,
+                                 llvm::Triple::Mesh});
       FD->setInvalidDecl();
     }
     break;
 
-  case HLSLShaderAttr::Compute:
-  case HLSLShaderAttr::Amplification:
-  case HLSLShaderAttr::Mesh:
+  case llvm::Triple::Compute:
+  case llvm::Triple::Amplification:
+  case llvm::Triple::Mesh:
     if (!FD->hasAttr<HLSLNumThreadsAttr>()) {
       Diag(FD->getLocation(), diag::err_hlsl_missing_numthreads)
-          << HLSLShaderAttr::ConvertShaderTypeToStr(ST);
+          << llvm::Triple::getEnvironmentTypeName(ST);
       FD->setInvalidDecl();
     }
     break;
+  default:
+    llvm_unreachable("Unhandled environment in triple");
   }
 
   for (ParmVarDecl *Param : FD->parameters()) {
@@ -268,14 +269,14 @@ void SemaHLSL::CheckSemanticAnnotation(
     const HLSLAnnotationAttr *AnnotationAttr) {
   auto *ShaderAttr = EntryPoint->getAttr<HLSLShaderAttr>();
   assert(ShaderAttr && "Entry point has no shader attribute");
-  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
+  llvm::Triple::EnvironmentType ST = ShaderAttr->getType();
 
   switch (AnnotationAttr->getKind()) {
   case attr::HLSLSV_DispatchThreadID:
   case attr::HLSLSV_GroupIndex:
-    if (ST == HLSLShaderAttr::Compute)
+    if (ST == llvm::Triple::Compute)
       return;
-    DiagnoseAttrStageMismatch(AnnotationAttr, ST, {HLSLShaderAttr::Compute});
+    DiagnoseAttrStageMismatch(AnnotationAttr, ST, {llvm::Triple::Compute});
     break;
   default:
     llvm_unreachable("Unknown HLSLAnnotationAttr");
@@ -283,16 +284,16 @@ void SemaHLSL::CheckSemanticAnnotation(
 }
 
 void SemaHLSL::DiagnoseAttrStageMismatch(
-    const Attr *A, HLSLShaderAttr::ShaderType Stage,
-    std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages) {
+    const Attr *A, llvm::Triple::EnvironmentType Stage,
+    std::initializer_list<llvm::Triple::EnvironmentType> AllowedStages) {
   SmallVector<StringRef, 8> StageStrings;
   llvm::transform(AllowedStages, std::back_inserter(StageStrings),
-                  [](HLSLShaderAttr::ShaderType ST) {
+                  [](llvm::Triple::EnvironmentType ST) {
                     return StringRef(
-                        HLSLShaderAttr::ConvertShaderTypeToStr(ST));
+                        HLSLShaderAttr::ConvertEnvironmentTypeToStr(ST));
                   });
   Diag(A->getLoc(), diag::err_hlsl_attr_unsupported_in_stage)
-      << A << HLSLShaderAttr::ConvertShaderTypeToStr(Stage)
+      << A << llvm::Triple::getEnvironmentTypeName(Stage)
       << (AllowedStages.size() != 1) << join(StageStrings, ", ");
 }
 
@@ -430,8 +431,8 @@ void SemaHLSL::handleShaderAttr(Decl *D, const ParsedAttr &AL) {
   if (!SemaRef.checkStringLiteralArgumentAttr(AL, 0, Str, &ArgLoc))
     return;
 
-  HLSLShaderAttr::ShaderType ShaderType;
-  if (!HLSLShaderAttr::ConvertStrToShaderType(Str, ShaderType)) {
+  llvm::Triple::EnvironmentType ShaderType;
+  if (!HLSLShaderAttr::ConvertStrToEnvironmentType(Str, ShaderType)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
         << AL << Str << ArgLoc;
     return;
@@ -549,16 +550,22 @@ class DiagnoseHLSLAvailability
   //
   // Maps FunctionDecl to an unsigned number that represents the set of shader
   // environments the function has been scanned for.
-  // Since HLSLShaderAttr::ShaderType enum is generated from Attr.td and is
-  // defined without any assigned values, it is guaranteed to be numbered
-  // sequentially from 0 up and we can use it to 'index' individual bits
-  // in the set.
+  // The llvm::Triple::EnvironmentType enum values for shader stages guaranteed
+  // to be numbered from llvm::Triple::Pixel to llvm::Triple::Amplification
+  // (verified by static_asserts in Triple.cpp), we can use it to index
+  // individual bits in the set, as long as we shift the values to start with 0
+  // by subtracting the value of llvm::Triple::Pixel first.
+  //
   // The N'th bit in the set will be set if the function has been scanned
-  // in shader environment whose ShaderType integer value equals N.
+  // in shader environment whose llvm::Triple::EnvironmentType integer value
+  // equals (llvm::Triple::Pixel + N).
+  //
   // For example, if a function has been scanned in compute and pixel stage
-  // environment, the value will be 0x21 (100001 binary) because
-  // (int)HLSLShaderAttr::ShaderType::Pixel == 1 and
-  // (int)HLSLShaderAttr::ShaderType::Compute == 5.
+  // environment, the value will be 0x21 (100001 binary) because:
+  //
+  //   (int)(llvm::Triple::Pixel - llvm::Triple::Pixel) == 0
+  //   (int)(llvm::Triple::Compute - llvm::Triple::Pixel) == 5
+  //
   // A FunctionDecl is mapped to 0 (or not included in the map) if it has not
   // been scanned in any environment.
   llvm::DenseMap<const FunctionDecl *, unsigned> ScannedDecls;
@@ -574,12 +581,16 @@ class DiagnoseHLSLAvailability
   bool ReportOnlyShaderStageIssues;
 
   // Helper methods for dealing with current stage context / environment
-  void SetShaderStageContext(HLSLShaderAttr::ShaderType ShaderType) {
+  void SetShaderStageContext(llvm::Triple::EnvironmentType ShaderType) {
     static_assert(sizeof(unsigned) >= 4);
-    assert((unsigned)ShaderType < 31); // 31 is reserved for "unknown"
-
-    CurrentShaderEnvironment = HLSLShaderAttr::getTypeAsEnvironment(ShaderType);
-    CurrentShaderStageBit = (1 << ShaderType);
+    assert(HLSLShaderAttr::isValidShaderType(ShaderType));
+    assert((unsigned)(ShaderType - llvm::Triple::Pixel) < 31 &&
+           "ShaderType is too big for this bitmap"); // 31 is reserved for
+                                                     // "unknown"
+
+    unsigned bitmapIndex = ShaderType - llvm::Triple::Pixel;
+    CurrentShaderEnvironment = ShaderType;
+    CurrentShaderStageBit = (1 << bitmapIndex);
   }
 
   void SetUnknownShaderStageContext() {
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 9ed3e8a0df025227bbac7f817c101a9bff7d09af..ed8b226a6b39f54709028a396c5bcebd69fd02d7 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -8063,6 +8063,11 @@ static void visitLocalsRetainedByInitializer(IndirectLocalPath &Path,
 enum PathLifetimeKind {
   /// Lifetime-extend along this path.
   Extend,
+  /// We should lifetime-extend, but we don't because (due to technical
+  /// limitations) we can't. This happens for default member initializers,
+  /// which we don't clone for every use, so we don't have a unique
+  /// MaterializeTemporaryExpr to update.
+  ShouldExtend,
   /// Do not lifetime extend along this path.
   NoExtend
 };
@@ -8074,7 +8079,7 @@ shouldLifetimeExtendThroughPath(const IndirectLocalPath &Path) {
   PathLifetimeKind Kind = PathLifetimeKind::Extend;
   for (auto Elem : Path) {
     if (Elem.Kind == IndirectLocalPathEntry::DefaultInit)
-      Kind = PathLifetimeKind::Extend;
+      Kind = PathLifetimeKind::ShouldExtend;
     else if (Elem.Kind != IndirectLocalPathEntry::LambdaCaptureInit)
       return PathLifetimeKind::NoExtend;
   }
@@ -8194,6 +8199,18 @@ void Sema::checkInitializerLifetime(const InitializedEntity &Entity,
                               ExtendingEntity->allocateManglingNumber());
         // Also visit the temporaries lifetime-extended by this initializer.
         return true;
+
+      case PathLifetimeKind::ShouldExtend:
+        // We're supposed to lifetime-extend the temporary along this path (per
+        // the resolution of DR1815), but we don't support that yet.
+        //
+        // FIXME: Properly handle this situation. Perhaps the easiest approach
+        // would be to clone the initializer expression on each use that would
+        // lifetime extend its temporaries.
+        Diag(DiagLoc, diag::warn_unsupported_lifetime_extension)
+            << RK << DiagRange;
+        break;
+
       case PathLifetimeKind::NoExtend:
         // If the path goes through the initialization of a variable or field,
         // it can't possibly reach a temporary created in this full-expression.
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 1320fd90b440bfbea4c14c1b158e22e052ea8734..97586a037eee4c17d7994f8e0480fd24f42b14c7 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -424,617 +424,736 @@ bool checkValidAfterDeviceType(
   S.Diag(DeviceTypeClause.getBeginLoc(), diag::note_acc_previous_clause_here);
   return true;
 }
-} // namespace
 
-SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
+class SemaOpenACCClauseVisitor {
+  SemaOpenACC &SemaRef;
+  ASTContext &Ctx;
+  ArrayRef<const OpenACCClause *> ExistingClauses;
+  bool NotImplemented = false;
 
-SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S,
-                                                    OpenACCDirectiveKind DK)
-    : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct),
-      DirKind(DK) {
-  // Compute constructs end up taking their 'loop'.
-  if (DirKind == OpenACCDirectiveKind::Parallel ||
-      DirKind == OpenACCDirectiveKind::Serial ||
-      DirKind == OpenACCDirectiveKind::Kernels) {
-    SemaRef.InsideComputeConstruct = true;
-    SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
+  OpenACCClause *isNotImplemented() {
+    NotImplemented = true;
+    return nullptr;
   }
-}
 
-SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
-  SemaRef.InsideComputeConstruct = WasInsideComputeConstruct;
-  if (DirKind == OpenACCDirectiveKind::Parallel ||
-      DirKind == OpenACCDirectiveKind::Serial ||
-      DirKind == OpenACCDirectiveKind::Kernels) {
-    assert(SemaRef.ParentlessLoopConstructs.empty() &&
-           "Didn't consume loop construct list?");
-    SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
-  }
-}
+public:
+  SemaOpenACCClauseVisitor(SemaOpenACC &S,
+                           ArrayRef<const OpenACCClause *> ExistingClauses)
+      : SemaRef(S), Ctx(S.getASTContext()), ExistingClauses(ExistingClauses) {}
+  // Once we've implemented everything, we shouldn't need this infrastructure.
+  // But in the meantime, we use this to help decide whether the clause was
+  // handled for this directive.
+  bool diagNotImplemented() { return NotImplemented; }
+
+  OpenACCClause *Visit(SemaOpenACC::OpenACCParsedClause &Clause) {
+    switch (Clause.getClauseKind()) {
+  case OpenACCClauseKind::Gang:
+  case OpenACCClauseKind::Worker:
+  case OpenACCClauseKind::Vector: {
+    // TODO OpenACC: These are only implemented enough for the 'seq' diagnostic,
+    // otherwise treats itself as unimplemented.  When we implement these, we
+    // can remove them from here.
 
-OpenACCClause *
-SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
-                         OpenACCParsedClause &Clause) {
-  if (Clause.getClauseKind() == OpenACCClauseKind::Invalid)
-    return nullptr;
+    // OpenACC 3.3 2.9:
+    // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
+    // appears.
+    const auto *Itr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
 
-  // Diagnose that we don't support this clause on this directive.
-  if (!doesClauseApplyToDirective(Clause.getDirectiveKind(),
-                                  Clause.getClauseKind())) {
-    Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment)
-        << Clause.getDirectiveKind() << Clause.getClauseKind();
-    return nullptr;
+    if (Itr != ExistingClauses.end()) {
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+          << Clause.getClauseKind() << (*Itr)->getClauseKind();
+      SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    }
+    return isNotImplemented();
   }
 
-  if (const auto *DevTypeClause =
-          llvm::find_if(ExistingClauses,
-                        [&](const OpenACCClause *C) {
-                          return isa<OpenACCDeviceTypeClause>(C);
-                        });
-      DevTypeClause != ExistingClauses.end()) {
-    if (checkValidAfterDeviceType(
-            *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
-      return nullptr;
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  case OpenACCClauseKind::CLAUSE_NAME:                                         \
+    return Visit##CLAUSE_NAME##Clause(Clause);
+#define CLAUSE_ALIAS(ALIAS, CLAUSE_NAME, DEPRECATED)                           \
+  case OpenACCClauseKind::ALIAS:                                               \
+  if (DEPRECATED)                                                              \
+    SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)   \
+        << Clause.getClauseKind() << OpenACCClauseKind::CLAUSE_NAME;           \
+  return Visit##CLAUSE_NAME##Clause(Clause);
+#include "clang/Basic/OpenACCClauses.def"
+    default:
+      return isNotImplemented();
+    }
+    llvm_unreachable("Invalid clause kind");
   }
 
-  switch (Clause.getClauseKind()) {
-  case OpenACCClauseKind::Default: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  OpenACCClause *Visit##CLAUSE_NAME##Clause(                                   \
+      SemaOpenACC::OpenACCParsedClause &Clause);
+#include "clang/Basic/OpenACCClauses.def"
+};
 
-    // Don't add an invalid clause to the AST.
-    if (Clause.getDefaultClauseKind() == OpenACCDefaultClauseKind::Invalid)
-      return nullptr;
-
-    // OpenACC 3.3, Section 2.5.4:
-    // At most one 'default' clause may appear, and it must have a value of
-    // either 'none' or 'present'.
-    // Second half of the sentence is diagnosed during parsing.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
-
-    return OpenACCDefaultClause::Create(
-        getASTContext(), Clause.getDefaultClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.getEndLoc());
-  }
+OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // Don't add an invalid clause to the AST.
+  if (Clause.getDefaultClauseKind() == OpenACCDefaultClauseKind::Invalid)
+    return nullptr;
 
-  case OpenACCClauseKind::If: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  // OpenACC 3.3, Section 2.5.4:
+  // At most one 'default' clause may appear, and it must have a value of
+  // either 'none' or 'present'.
+  // Second half of the sentence is diagnosed during parsing.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+  return OpenACCDefaultClause::Create(
+      Ctx, Clause.getDefaultClauseKind(), Clause.getBeginLoc(),
+      Clause.getLParenLoc(), Clause.getEndLoc());
+}
 
-    // The parser has ensured that we have a proper condition expr, so there
-    // isn't really much to do here.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitIfClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    // If the 'if' clause is true, it makes the 'self' clause have no effect,
-    // diagnose that here.
-    // TODO OpenACC: When we add these two to other constructs, we might not
-    // want to warn on this (for example, 'update').
-    const auto *Itr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSelfClause>);
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
+  // The parser has ensured that we have a proper condition expr, so there
+  // isn't really much to do here.
 
-    return OpenACCIfClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getConditionExpr(), Clause.getEndLoc());
+  // If the 'if' clause is true, it makes the 'self' clause have no effect,
+  // diagnose that here.
+  // TODO OpenACC: When we add these two to other constructs, we might not
+  // want to warn on this (for example, 'update').
+  const auto *Itr =
+      llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSelfClause>);
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
   }
 
-  case OpenACCClauseKind::Self: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
-
-    // TODO OpenACC: When we implement this for 'update', this takes a
-    // 'var-list' instead of a condition expression, so semantics/handling has
-    // to happen differently here.
-
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+  return OpenACCIfClause::Create(Ctx, Clause.getBeginLoc(),
+                                 Clause.getLParenLoc(),
+                                 Clause.getConditionExpr(), Clause.getEndLoc());
+}
 
-    // If the 'if' clause is true, it makes the 'self' clause have no effect,
-    // diagnose that here.
-    // TODO OpenACC: When we add these two to other constructs, we might not
-    // want to warn on this (for example, 'update').
-    const auto *Itr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCIfClause>);
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
+OpenACCClause *SemaOpenACCClauseVisitor::VisitSelfClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // TODO OpenACC: When we implement this for 'update', this takes a
+  // 'var-list' instead of a condition expression, so semantics/handling has
+  // to happen differently here.
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    return OpenACCSelfClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getConditionExpr(), Clause.getEndLoc());
+  // If the 'if' clause is true, it makes the 'self' clause have no effect,
+  // diagnose that here.
+  // TODO OpenACC: When we add these two to other constructs, we might not
+  // want to warn on this (for example, 'update').
+  const auto *Itr =
+      llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCIfClause>);
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
   }
-  case OpenACCClauseKind::NumGangs: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  return OpenACCSelfClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.getConditionExpr(), Clause.getEndLoc());
+}
 
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    if (Clause.getIntExprs().empty())
-      Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args)
-          << /*NoArgs=*/0;
-
-    unsigned MaxArgs =
-        (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel ||
-         Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop)
-            ? 3
-            : 1;
-    if (Clause.getIntExprs().size() > MaxArgs)
-      Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args)
-          << /*NoArgs=*/1 << Clause.getDirectiveKind() << MaxArgs
+  // num_gangs requires at least 1 int expr in all forms.  Diagnose here, but
+  // allow us to continue, an empty clause might be useful for future
+  // diagnostics.
+  if (Clause.getIntExprs().empty())
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args)
+        << /*NoArgs=*/0;
+
+  unsigned MaxArgs =
+      (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel ||
+       Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop)
+          ? 3
+          : 1;
+  // The max number of args differs between parallel and other constructs.
+  // Again, allow us to continue for the purposes of future diagnostics.
+  if (Clause.getIntExprs().size() > MaxArgs)
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args)
+        << /*NoArgs=*/1 << Clause.getDirectiveKind() << MaxArgs
+        << Clause.getIntExprs().size();
+
+  // OpenACC 3.3 Section 2.5.4:
+  // A reduction clause may not appear on a parallel construct with a
+  // num_gangs clause that has more than one argument.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel &&
+      Clause.getIntExprs().size() > 1) {
+    auto *Parallel =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
+
+    if (Parallel != ExistingClauses.end()) {
+      SemaRef.Diag(Clause.getBeginLoc(),
+                   diag::err_acc_reduction_num_gangs_conflict)
           << Clause.getIntExprs().size();
-
-    // OpenACC 3.3 Section 2.5.4:
-    // A reduction clause may not appear on a parallel construct with a
-    // num_gangs clause that has more than one argument.
-    if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel &&
-        Clause.getIntExprs().size() > 1) {
-      auto *Parallel =
-          llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
-
-      if (Parallel != ExistingClauses.end()) {
-        Diag(Clause.getBeginLoc(), diag::err_acc_reduction_num_gangs_conflict)
-            << Clause.getIntExprs().size();
-        Diag((*Parallel)->getBeginLoc(), diag::note_acc_previous_clause_here);
-        return nullptr;
-      }
+      SemaRef.Diag((*Parallel)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
     }
-
-    // Create the AST node for the clause even if the number of expressions is
-    // incorrect.
-    return OpenACCNumGangsClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getIntExprs(), Clause.getEndLoc());
-    break;
   }
-  case OpenACCClauseKind::NumWorkers: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  return OpenACCNumGangsClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs(),
+      Clause.getEndLoc());
+}
 
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    assert(Clause.getIntExprs().size() == 1 &&
-           "Invalid number of expressions for NumWorkers");
-    return OpenACCNumWorkersClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getIntExprs()[0], Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::VectorLength: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  assert(Clause.getIntExprs().size() == 1 &&
+         "Invalid number of expressions for NumWorkers");
+  return OpenACCNumWorkersClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs()[0],
+      Clause.getEndLoc());
+}
 
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorLengthClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    assert(Clause.getIntExprs().size() == 1 &&
-           "Invalid number of expressions for VectorLength");
-    return OpenACCVectorLengthClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getIntExprs()[0], Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Async: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  assert(Clause.getIntExprs().size() == 1 &&
+         "Invalid number of expressions for NumWorkers");
+  return OpenACCVectorLengthClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs()[0],
+      Clause.getEndLoc());
+}
 
-    // There is no prose in the standard that says duplicates aren't allowed,
-    // but this diagnostic is present in other compilers, as well as makes
-    // sense.
-    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
-      return nullptr;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitAsyncClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // There is no prose in the standard that says duplicates aren't allowed,
+  // but this diagnostic is present in other compilers, as well as makes
+  // sense.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
 
-    assert(Clause.getNumIntExprs() < 2 &&
-           "Invalid number of expressions for Async");
+  assert(Clause.getNumIntExprs() < 2 &&
+         "Invalid number of expressions for Async");
+  return OpenACCAsyncClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr,
+      Clause.getEndLoc());
+}
 
-    return OpenACCAsyncClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr,
-        Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Private: {
-    // Restrictions only properly implemented on 'compute' and 'loop'
-    // constructs, and 'compute'/'loop' constructs are the only construct that
-    // can do anything with this yet, so skip/treat as unimplemented in this
-    // case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitPrivateClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' and 'loop'
+  // constructs, and 'compute'/'loop' constructs are the only construct that
+  // can do anything with this yet, so skip/treat as unimplemented in this
+  // case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCPrivateClause::Create(Ctx, Clause.getBeginLoc(),
+                                      Clause.getLParenLoc(),
+                                      Clause.getVarList(), Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitFirstPrivateClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCFirstPrivateClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getVarList(),
+      Clause.getEndLoc());
+}
 
-    return OpenACCPrivateClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::FirstPrivate: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitNoCreateClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCNoCreateClause::Create(Ctx, Clause.getBeginLoc(),
+                                       Clause.getLParenLoc(),
+                                       Clause.getVarList(), Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitPresentClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCPresentClause::Create(Ctx, Clause.getBeginLoc(),
+                                      Clause.getLParenLoc(),
+                                      Clause.getVarList(), Clause.getEndLoc());
+}
 
-    return OpenACCFirstPrivateClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::NoCreate: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCCopyClause::Create(
+      Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.getVarList(), Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyInClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCCopyInClause::Create(
+      Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.isReadOnly(), Clause.getVarList(), Clause.getEndLoc());
+}
 
-    return OpenACCNoCreateClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Present: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyOutClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCCopyOutClause::Create(
+      Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.isZero(), Clause.getVarList(), Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitCreateClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+  // ActOnVar ensured that everything is a valid variable reference, so there
+  // really isn't anything to do here. GCC does some duplicate-finding, though
+  // it isn't apparent in the standard where this is justified.
+
+  return OpenACCCreateClause::Create(
+      Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.isZero(), Clause.getVarList(), Clause.getEndLoc());
+}
 
-    return OpenACCPresentClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::PresentOrCopy:
-  case OpenACCClauseKind::PCopy:
-    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
-        << Clause.getClauseKind() << OpenACCClauseKind::Copy;
-    LLVM_FALLTHROUGH;
-  case OpenACCClauseKind::Copy: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitAttachClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // ActOnVar ensured that everything is a valid variable reference, but we
+  // still have to make sure it is a pointer type.
+  llvm::SmallVector<Expr *> VarList{Clause.getVarList().begin(),
+                                    Clause.getVarList().end()};
+  VarList.erase(std::remove_if(VarList.begin(), VarList.end(),
+                               [&](Expr *E) {
+                                 return SemaRef.CheckVarIsPointerType(
+                                     OpenACCClauseKind::Attach, E);
+                               }),
+                VarList.end());
+  Clause.setVarListDetails(VarList,
+                           /*IsReadOnly=*/false, /*IsZero=*/false);
+  return OpenACCAttachClause::Create(Ctx, Clause.getBeginLoc(),
+                                     Clause.getLParenLoc(), Clause.getVarList(),
+                                     Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitDevicePtrClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // ActOnVar ensured that everything is a valid variable reference, but we
+  // still have to make sure it is a pointer type.
+  llvm::SmallVector<Expr *> VarList{Clause.getVarList().begin(),
+                                    Clause.getVarList().end()};
+  VarList.erase(std::remove_if(VarList.begin(), VarList.end(),
+                               [&](Expr *E) {
+                                 return SemaRef.CheckVarIsPointerType(
+                                     OpenACCClauseKind::DevicePtr, E);
+                               }),
+                VarList.end());
+  Clause.setVarListDetails(VarList,
+                           /*IsReadOnly=*/false, /*IsZero=*/false);
+
+  return OpenACCDevicePtrClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getVarList(),
+      Clause.getEndLoc());
+}
 
-    return OpenACCCopyClause::Create(
-        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::PresentOrCopyIn:
-  case OpenACCClauseKind::PCopyIn:
-    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
-        << Clause.getClauseKind() << OpenACCClauseKind::CopyIn;
-    LLVM_FALLTHROUGH;
-  case OpenACCClauseKind::CopyIn: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  return OpenACCWaitClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getDevNumExpr(),
+      Clause.getQueuesLoc(), Clause.getQueueIdExprs(), Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' and 'loop'
+  // constructs, and 'compute'/'loop' constructs are the only construct that
+  // can do anything with this yet, so skip/treat as unimplemented in this
+  // case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // TODO OpenACC: Once we get enough of the CodeGen implemented that we have
+  // a source for the list of valid architectures, we need to warn on unknown
+  // identifiers here.
+
+  return OpenACCDeviceTypeClause::Create(
+      Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+      Clause.getDeviceTypeArchitectures(), Clause.getEndLoc());
+}
 
-    return OpenACCCopyInClause::Create(
-        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.isReadOnly(), Clause.getVarList(),
-        Clause.getEndLoc());
+OpenACCClause *SemaOpenACCClauseVisitor::VisitAutoClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'loop' constructs, and it is
+  // the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the combined constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // OpenACC 3.3 2.9:
+  // Only one of the seq, independent, and auto clauses may appear.
+  const auto *Itr =
+      llvm::find_if(ExistingClauses,
+                    llvm::IsaPred<OpenACCIndependentClause, OpenACCSeqClause>);
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
+        << Clause.getClauseKind() << Clause.getDirectiveKind();
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    return nullptr;
   }
-  case OpenACCClauseKind::PresentOrCopyOut:
-  case OpenACCClauseKind::PCopyOut:
-    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
-        << Clause.getClauseKind() << OpenACCClauseKind::CopyOut;
-    LLVM_FALLTHROUGH;
-  case OpenACCClauseKind::CopyOut: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
 
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
+  return OpenACCAutoClause::Create(Ctx, Clause.getBeginLoc(),
+                                   Clause.getEndLoc());
+}
 
-    return OpenACCCopyOutClause::Create(
-        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.isZero(), Clause.getVarList(),
-        Clause.getEndLoc());
+OpenACCClause *SemaOpenACCClauseVisitor::VisitIndependentClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'loop' constructs, and it is
+  // the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the combined constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // OpenACC 3.3 2.9:
+  // Only one of the seq, independent, and auto clauses may appear.
+  const auto *Itr = llvm::find_if(
+      ExistingClauses, llvm::IsaPred<OpenACCAutoClause, OpenACCSeqClause>);
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
+        << Clause.getClauseKind() << Clause.getDirectiveKind();
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    return nullptr;
   }
-  case OpenACCClauseKind::PresentOrCreate:
-  case OpenACCClauseKind::PCreate:
-    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
-        << Clause.getClauseKind() << OpenACCClauseKind::Create;
-    LLVM_FALLTHROUGH;
-  case OpenACCClauseKind::Create: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
-
-    // ActOnVar ensured that everything is a valid variable reference, so there
-    // really isn't anything to do here. GCC does some duplicate-finding, though
-    // it isn't apparent in the standard where this is justified.
 
-    return OpenACCCreateClause::Create(getASTContext(), Clause.getClauseKind(),
-                                       Clause.getBeginLoc(),
-                                       Clause.getLParenLoc(), Clause.isZero(),
-                                       Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Attach: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  return OpenACCIndependentClause::Create(Ctx, Clause.getBeginLoc(),
+                                          Clause.getEndLoc());
+}
 
-    // ActOnVar ensured that everything is a valid variable reference, but we
-    // still have to make sure it is a pointer type.
-    llvm::SmallVector<Expr *> VarList{Clause.getVarList().begin(),
-                                      Clause.getVarList().end()};
-    VarList.erase(std::remove_if(VarList.begin(), VarList.end(), [&](Expr *E) {
-      return CheckVarIsPointerType(OpenACCClauseKind::Attach, E);
-    }), VarList.end());
-    Clause.setVarListDetails(VarList,
-                             /*IsReadOnly=*/false, /*IsZero=*/false);
-
-    return OpenACCAttachClause::Create(getASTContext(), Clause.getBeginLoc(),
-                                       Clause.getLParenLoc(),
-                                       Clause.getVarList(), Clause.getEndLoc());
+OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'loop' constructs, and it is
+  // the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the combined constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // OpenACC 3.3 2.9:
+  // Only one of the seq, independent, and auto clauses may appear.
+  const auto *Itr =
+      llvm::find_if(ExistingClauses,
+                    llvm::IsaPred<OpenACCAutoClause, OpenACCIndependentClause>);
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
+        << Clause.getClauseKind() << Clause.getDirectiveKind();
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    return nullptr;
   }
-  case OpenACCClauseKind::DevicePtr: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
 
-    // ActOnVar ensured that everything is a valid variable reference, but we
-    // still have to make sure it is a pointer type.
-    llvm::SmallVector<Expr *> VarList{Clause.getVarList().begin(),
-                                      Clause.getVarList().end()};
-    VarList.erase(std::remove_if(VarList.begin(), VarList.end(), [&](Expr *E) {
-      return CheckVarIsPointerType(OpenACCClauseKind::DevicePtr, E);
-    }), VarList.end());
-    Clause.setVarListDetails(VarList,
-                             /*IsReadOnly=*/false, /*IsZero=*/false);
-
-    return OpenACCDevicePtrClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getVarList(), Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Wait: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
+  // OpenACC 3.3 2.9:
+  // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
+  // appears.
+  Itr = llvm::find_if(ExistingClauses,
+                      llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
+                                    OpenACCVectorClause>);
 
-    return OpenACCWaitClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(),
-        Clause.getEndLoc());
+  if (Itr != ExistingClauses.end()) {
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+        << Clause.getClauseKind() << (*Itr)->getClauseKind();
+    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    return nullptr;
   }
-  case OpenACCClauseKind::DType:
-  case OpenACCClauseKind::DeviceType: {
-    // Restrictions only properly implemented on 'compute' and 'loop'
-    // constructs, and 'compute'/'loop' constructs are the only construct that
-    // can do anything with this yet, so skip/treat as unimplemented in this
-    // case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-      break;
 
-    // TODO OpenACC: Once we get enough of the CodeGen implemented that we have
-    // a source for the list of valid architectures, we need to warn on unknown
-    // identifiers here.
-
-    return OpenACCDeviceTypeClause::Create(
-        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.getDeviceTypeArchitectures(),
-        Clause.getEndLoc());
-  }
-  case OpenACCClauseKind::Auto: {
-    // Restrictions only properly implemented on 'loop' constructs, and it is
-    // the only construct that can do anything with this, so skip/treat as
-    // unimplemented for the combined constructs.
-    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-      break;
+  // TODO OpenACC: 2.9 ~ line 2010 specifies that the associated loop has some
+  // restrictions when there is a 'seq' clause in place. We probably need to
+  // implement that.
+  return OpenACCSeqClause::Create(Ctx, Clause.getBeginLoc(),
+                                  Clause.getEndLoc());
+}
 
-    // OpenACC 3.3 2.9:
-    // Only one of the seq, independent, and auto clauses may appear.
-    const auto *Itr = llvm::find_if(
-        ExistingClauses,
-        llvm::IsaPred<OpenACCIndependentClause, OpenACCSeqClause>);
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-          << Clause.getClauseKind() << Clause.getDirectiveKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  // Restrictions only properly implemented on 'compute' constructs, and
+  // 'compute' constructs are the only construct that can do anything with
+  // this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+    return isNotImplemented();
+
+  // OpenACC 3.3 Section 2.5.4:
+  // A reduction clause may not appear on a parallel construct with a
+  // num_gangs clause that has more than one argument.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel) {
+    auto NumGangsClauses = llvm::make_filter_range(
+        ExistingClauses, llvm::IsaPred<OpenACCNumGangsClause>);
+
+    for (auto *NGC : NumGangsClauses) {
+      unsigned NumExprs =
+          cast<OpenACCNumGangsClause>(NGC)->getIntExprs().size();
+
+      if (NumExprs > 1) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_reduction_num_gangs_conflict)
+            << NumExprs;
+        SemaRef.Diag(NGC->getBeginLoc(), diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
     }
-
-    return OpenACCAutoClause::Create(getASTContext(), Clause.getBeginLoc(),
-                                     Clause.getEndLoc());
   }
-  case OpenACCClauseKind::Independent: {
-    // Restrictions only properly implemented on 'loop' constructs, and it is
-    // the only construct that can do anything with this, so skip/treat as
-    // unimplemented for the combined constructs.
-    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-      break;
 
-    // OpenACC 3.3 2.9:
-    // Only one of the seq, independent, and auto clauses may appear.
-    const auto *Itr = llvm::find_if(
-        ExistingClauses, llvm::IsaPred<OpenACCAutoClause, OpenACCSeqClause>);
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-          << Clause.getClauseKind() << Clause.getDirectiveKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
+  SmallVector<Expr *> ValidVars;
+
+  for (Expr *Var : Clause.getVarList()) {
+    ExprResult Res = SemaRef.CheckReductionVar(Var);
 
-    return OpenACCIndependentClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getEndLoc());
+    if (Res.isUsable())
+      ValidVars.push_back(Res.get());
   }
-  case OpenACCClauseKind::Seq: {
-    // Restrictions only properly implemented on 'loop' constructs, and it is
-    // the only construct that can do anything with this, so skip/treat as
-    // unimplemented for the combined constructs.
-    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-      break;
 
-    // OpenACC 3.3 2.9:
-    // Only one of the seq, independent, and auto clauses may appear.
-    const auto *Itr = llvm::find_if(
-        ExistingClauses,
-        llvm::IsaPred<OpenACCAutoClause, OpenACCIndependentClause>);
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-          << Clause.getClauseKind() << Clause.getDirectiveKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
+  return OpenACCReductionClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getReductionOp(),
+      ValidVars, Clause.getEndLoc());
+}
 
-    // OpenACC 3.3 2.9:
-    // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
-    // appears.
-    Itr = llvm::find_if(ExistingClauses,
-                        llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
-                                      OpenACCVectorClause>);
+} // namespace
 
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
-          << Clause.getClauseKind() << (*Itr)->getClauseKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
+SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
 
-    // TODO OpenACC: 2.9 ~ line 2010 specifies that the associated loop has some
-    // restrictions when there is a 'seq' clause in place. We probably need to
-    // implement that.
-    return OpenACCSeqClause::Create(getASTContext(), Clause.getBeginLoc(),
-                                    Clause.getEndLoc());
+SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S,
+                                                    OpenACCDirectiveKind DK)
+    : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct),
+      DirKind(DK) {
+  // Compute constructs end up taking their 'loop'.
+  if (DirKind == OpenACCDirectiveKind::Parallel ||
+      DirKind == OpenACCDirectiveKind::Serial ||
+      DirKind == OpenACCDirectiveKind::Kernels) {
+    SemaRef.InsideComputeConstruct = true;
+    SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
   }
-  case OpenACCClauseKind::Gang:
-  case OpenACCClauseKind::Worker:
-  case OpenACCClauseKind::Vector: {
-    // OpenACC 3.3 2.9:
-    // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
-    // appears.
-    const auto *Itr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
+}
 
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
-          << Clause.getClauseKind() << (*Itr)->getClauseKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    }
-    // Not yet implemented, so immediately drop to the 'not yet implemented'
-    // diagnostic.
-    break;
+SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
+  SemaRef.InsideComputeConstruct = WasInsideComputeConstruct;
+  if (DirKind == OpenACCDirectiveKind::Parallel ||
+      DirKind == OpenACCDirectiveKind::Serial ||
+      DirKind == OpenACCDirectiveKind::Kernels) {
+    assert(SemaRef.ParentlessLoopConstructs.empty() &&
+           "Didn't consume loop construct list?");
+    SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
   }
-  case OpenACCClauseKind::Reduction: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
-
-    // OpenACC 3.3 Section 2.5.4:
-    // A reduction clause may not appear on a parallel construct with a
-    // num_gangs clause that has more than one argument.
-    if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel) {
-      auto NumGangsClauses = llvm::make_filter_range(
-          ExistingClauses, llvm::IsaPred<OpenACCNumGangsClause>);
-
-      for (auto *NGC : NumGangsClauses) {
-        unsigned NumExprs =
-            cast<OpenACCNumGangsClause>(NGC)->getIntExprs().size();
-
-        if (NumExprs > 1) {
-          Diag(Clause.getBeginLoc(), diag::err_acc_reduction_num_gangs_conflict)
-              << NumExprs;
-          Diag(NGC->getBeginLoc(), diag::note_acc_previous_clause_here);
-          return nullptr;
-        }
-      }
-    }
-
-    SmallVector<Expr *> ValidVars;
-
-    for (Expr *Var : Clause.getVarList()) {
-      ExprResult Res = CheckReductionVar(Var);
+}
 
-      if (Res.isUsable())
-        ValidVars.push_back(Res.get());
-    }
+OpenACCClause *
+SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
+                         OpenACCParsedClause &Clause) {
+  if (Clause.getClauseKind() == OpenACCClauseKind::Invalid)
+    return nullptr;
 
-    return OpenACCReductionClause::Create(
-        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
-        Clause.getReductionOp(), ValidVars, Clause.getEndLoc());
+  // Diagnose that we don't support this clause on this directive.
+  if (!doesClauseApplyToDirective(Clause.getDirectiveKind(),
+                                  Clause.getClauseKind())) {
+    Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment)
+        << Clause.getDirectiveKind() << Clause.getClauseKind();
+    return nullptr;
   }
-  default:
-    break;
+
+  if (const auto *DevTypeClause =
+          llvm::find_if(ExistingClauses,
+                        [&](const OpenACCClause *C) {
+                          return isa<OpenACCDeviceTypeClause>(C);
+                        });
+      DevTypeClause != ExistingClauses.end()) {
+    if (checkValidAfterDeviceType(
+            *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
+      return nullptr;
   }
 
-  Diag(Clause.getBeginLoc(), diag::warn_acc_clause_unimplemented)
-      << Clause.getClauseKind();
-  return nullptr;
+  SemaOpenACCClauseVisitor Visitor{*this, ExistingClauses};
+  OpenACCClause *Result = Visitor.Visit(Clause);
+  assert((!Result || Result->getClauseKind() == Clause.getClauseKind()) &&
+         "Created wrong clause?");
+
+  if (Visitor.diagNotImplemented())
+    Diag(Clause.getBeginLoc(), diag::warn_acc_clause_unimplemented)
+        << Clause.getClauseKind();
+
+  return Result;
+
+  //  switch (Clause.getClauseKind()) {
+  //  case OpenACCClauseKind::PresentOrCopy:
+  //  case OpenACCClauseKind::PCopy:
+  //    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
+  //        << Clause.getClauseKind() << OpenACCClauseKind::Copy;
+  //    LLVM_FALLTHROUGH;
+  //  case OpenACCClauseKind::PresentOrCreate:
+  //  case OpenACCClauseKind::PCreate:
+  //    Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name)
+  //        << Clause.getClauseKind() << OpenACCClauseKind::Create;
+  //    LLVM_FALLTHROUGH;
+  //
+  //
+  //
+  //
+  //  case OpenACCClauseKind::DType:
+  //
+  //
+  //
+  //
+  //
+  //
+  //
+  //
+  //  case OpenACCClauseKind::Gang:
+  //  case OpenACCClauseKind::Worker:
+  //  case OpenACCClauseKind::Vector: {
+  //    // OpenACC 3.3 2.9:
+  //    // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq'
+  //    clause
+  //    // appears.
+  //    const auto *Itr =
+  //        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
+  //
+  //    if (Itr != ExistingClauses.end()) {
+  //      Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+  //          << Clause.getClauseKind() << (*Itr)->getClauseKind();
+  //      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+  //    }
+  //    // Not yet implemented, so immediately drop to the 'not yet implemented'
+  //    // diagnostic.
+  //    break;
+  //  }
+  //  */
+
 }
 
 /// OpenACC 3.3 section 2.5.15:
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 1011db2d2830dcd41919b229d339b94c03d99e5f..befeb38e1fe5bc63ec3b2a02b68e12c93f4f599c 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -712,13 +712,6 @@ DeduceTemplateSpecArguments(Sema &S, TemplateParameterList *TemplateParams,
     if (const auto *TD = TNA.getAsTemplateDecl(); TD && TD->isTypeAlias())
       return TemplateDeductionResult::Success;
 
-    // Perform template argument deduction for the template name.
-    if (auto Result =
-            DeduceTemplateArguments(S, TemplateParams, TNP, TNA, Info,
-                                    SA->template_arguments(), Deduced);
-        Result != TemplateDeductionResult::Success)
-      return Result;
-
     // FIXME: To preserve sugar, the TST needs to carry sugared resolved
     // arguments.
     ArrayRef<TemplateArgument> AResolved =
@@ -726,6 +719,12 @@ DeduceTemplateSpecArguments(Sema &S, TemplateParameterList *TemplateParams,
             ->castAs<TemplateSpecializationType>()
             ->template_arguments();
 
+    // Perform template argument deduction for the template name.
+    if (auto Result = DeduceTemplateArguments(S, TemplateParams, TNP, TNA, Info,
+                                              AResolved, Deduced);
+        Result != TemplateDeductionResult::Success)
+      return Result;
+
     // Perform template argument deduction on each template
     // argument. Ignore any missing/extra arguments, since they could be
     // filled in by default arguments.
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 95dd356d48bedd2e9230e1d2b569a363799aed28..3bfda09d5f80fccf4ffd473927ce7f45dd82fbf3 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -14172,13 +14172,6 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
     if (TransformExprs(E->getArgs(), E->getNumArgs(), true, Args,
                        &ArgumentChanged))
       return ExprError();
-
-    if (E->isListInitialization() && !E->isStdInitListInitialization()) {
-      ExprResult Res = RebuildInitList(E->getBeginLoc(), Args, E->getEndLoc());
-      if (Res.isInvalid())
-        return ExprError();
-      Args = {Res.get()};
-    }
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -14190,9 +14183,12 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
     return SemaRef.MaybeBindToTemporary(E);
   }
 
+  // FIXME: We should just pass E->isListInitialization(), but we're not
+  // prepared to handle list-initialization without a child InitListExpr.
   SourceLocation LParenLoc = T->getTypeLoc().getEndLoc();
   return getDerived().RebuildCXXTemporaryObjectExpr(
-      T, LParenLoc, Args, E->getEndLoc(), E->isListInitialization());
+      T, LParenLoc, Args, E->getEndLoc(),
+      /*ListInitialization=*/LParenLoc.isInvalid());
 }
 
 template<typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 33cea32c3be6126351ee9fe50b1950cf1ff13f7c..59338b44db32f34932e3abb5e55932b7392210fd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
 
     unsigned NumFileDecls = Record[7];
     if (NumFileDecls && ContextObj) {
-      const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6];
+      const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6];
       assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?");
       FileDeclIDs[FID] =
           FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls));
@@ -3377,26 +3377,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
             "duplicate DECL_OFFSET record in AST file");
       F.DeclOffsets = (const DeclOffset *)Blob.data();
       F.LocalNumDecls = Record[0];
-      unsigned LocalBaseDeclID = Record[1];
-      F.BaseDeclID = getTotalNumDecls();
-
-      if (F.LocalNumDecls > 0) {
-        // Introduce the global -> local mapping for declarations within this
-        // module.
-        GlobalDeclMap.insert(std::make_pair(
-            GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F));
-
-        // Introduce the local -> global mapping for declarations within this
-        // module.
-        F.DeclRemap.insertOrReplace(
-          std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID));
-
-        // Introduce the global -> local mapping for declarations within this
-        // module.
-        F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID;
+      F.BaseDeclIndex = getTotalNumDecls();
 
+      if (F.LocalNumDecls > 0)
         DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls);
-      }
+
       break;
     }
 
@@ -3631,7 +3616,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       break;
 
     case FILE_SORTED_DECLS:
-      F.FileSortedDecls = (const LocalDeclID *)Blob.data();
+      F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data();
       F.NumFileSortedDecls = Record[0];
       break;
 
@@ -4058,7 +4043,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
   RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap);
   RemapBuilder SubmoduleRemap(F.SubmoduleRemap);
   RemapBuilder SelectorRemap(F.SelectorRemap);
-  RemapBuilder DeclRemap(F.DeclRemap);
   RemapBuilder TypeRemap(F.TypeRemap);
 
   auto &ImportedModuleVector = F.TransitiveImports;
@@ -4097,8 +4081,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
         endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t SelectorIDOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(Data);
-    uint32_t DeclIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t TypeIndexOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(Data);
 
@@ -4116,11 +4098,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
               PreprocessedEntityRemap);
     mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap);
     mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap);
-    mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap);
     mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap);
-
-    // Global -> local mappings.
-    F.GlobalToLocalDeclIDs[OM] = DeclIDOffset;
   }
 }
 
@@ -4645,7 +4623,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type,
   // that we load any additional categories.
   if (ContextObj) {
     for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) {
-      loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()),
+      loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(),
                          ObjCClassesLoaded[I], PreviousGeneration);
     }
   }
@@ -7644,18 +7622,25 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
 
 GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F,
                                         LocalDeclID LocalID) const {
-  DeclID ID = LocalID.get();
-  if (ID < NUM_PREDEF_DECL_IDS)
-    return GlobalDeclID(ID);
+  if (LocalID.get() < NUM_PREDEF_DECL_IDS)
+    return GlobalDeclID(LocalID.get());
+
+  unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex();
+  DeclID ID = LocalID.getLocalDeclIndex();
 
   if (!F.ModuleOffsetMap.empty())
     ReadModuleOffsetMap(F);
 
-  ContinuousRangeMap<DeclID, int, 2>::iterator I =
-      F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS);
-  assert(I != F.DeclRemap.end() && "Invalid index into decl index remap");
+  ModuleFile *OwningModuleFile =
+      OwningModuleFileIndex == 0
+          ? &F
+          : F.TransitiveImports[OwningModuleFileIndex - 1];
+
+  if (OwningModuleFileIndex == 0)
+    ID -= NUM_PREDEF_DECL_IDS;
 
-  return GlobalDeclID(ID + I->second);
+  uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1;
+  return GlobalDeclID(ID, NewModuleFileIndex);
 }
 
 bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const {
@@ -7663,31 +7648,33 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const {
   if (ID.get() < NUM_PREDEF_DECL_IDS)
     return false;
 
-  return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID &&
-         ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls;
+  unsigned ModuleFileIndex = ID.getModuleFileIndex();
+  return M.Index == ModuleFileIndex - 1;
+}
+
+ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const {
+  // Predefined decls aren't from any module.
+  if (ID.get() < NUM_PREDEF_DECL_IDS)
+    return nullptr;
+
+  uint64_t ModuleFileIndex = ID.getModuleFileIndex();
+  assert(ModuleFileIndex && "Untranslated Local Decl?");
+
+  return &getModuleManager()[ModuleFileIndex - 1];
 }
 
-ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) {
+ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const {
   if (!D->isFromASTFile())
     return nullptr;
-  GlobalDeclMapType::const_iterator I =
-      GlobalDeclMap.find(GlobalDeclID(D->getGlobalID()));
-  assert(I != GlobalDeclMap.end() && "Corrupted global declaration map");
-  return I->second;
+
+  return getOwningModuleFile(D->getGlobalID());
 }
 
 SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) {
   if (ID.get() < NUM_PREDEF_DECL_IDS)
     return SourceLocation();
 
-  unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS;
-
-  if (Index > DeclsLoaded.size()) {
-    Error("declaration ID out-of-range for AST file");
-    return SourceLocation();
-  }
-
-  if (Decl *D = DeclsLoaded[Index])
+  if (Decl *D = GetExistingDecl(ID))
     return D->getLocation();
 
   SourceLocation Loc;
@@ -7754,8 +7741,19 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) {
   llvm_unreachable("PredefinedDeclIDs unknown enum value");
 }
 
+unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const {
+  ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID);
+  if (!OwningModuleFile) {
+    assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?");
+    return GlobalID.get();
+  }
+
+  return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex();
+}
+
 Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) {
   assert(ContextObj && "reading decl with no AST context");
+
   if (ID.get() < NUM_PREDEF_DECL_IDS) {
     Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID);
     if (D) {
@@ -7768,7 +7766,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) {
     return D;
   }
 
-  unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS;
+  unsigned Index = translateGlobalDeclIDToIndex(ID);
 
   if (Index >= DeclsLoaded.size()) {
     assert(0 && "declaration ID out-of-range for AST file");
@@ -7783,7 +7781,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) {
   if (ID.get() < NUM_PREDEF_DECL_IDS)
     return GetExistingDecl(ID);
 
-  unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS;
+  unsigned Index = translateGlobalDeclIDToIndex(ID);
 
   if (Index >= DeclsLoaded.size()) {
     assert(0 && "declaration ID out-of-range for AST file");
@@ -7802,20 +7800,31 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) {
 
 LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M,
                                                        GlobalDeclID GlobalID) {
-  DeclID ID = GlobalID.get();
-  if (ID < NUM_PREDEF_DECL_IDS)
+  if (GlobalID.get() < NUM_PREDEF_DECL_IDS)
+    return LocalDeclID(GlobalID.get());
+
+  if (!M.ModuleOffsetMap.empty())
+    ReadModuleOffsetMap(M);
+
+  ModuleFile *Owner = getOwningModuleFile(GlobalID);
+  DeclID ID = GlobalID.getLocalDeclIndex();
+
+  if (Owner == &M) {
+    ID += NUM_PREDEF_DECL_IDS;
     return LocalDeclID(ID);
+  }
 
-  GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID);
-  assert(I != GlobalDeclMap.end() && "Corrupted global declaration map");
-  ModuleFile *Owner = I->second;
+  uint64_t OrignalModuleFileIndex = 0;
+  for (unsigned I = 0; I < M.TransitiveImports.size(); I++)
+    if (M.TransitiveImports[I] == Owner) {
+      OrignalModuleFileIndex = I + 1;
+      break;
+    }
 
-  llvm::DenseMap<ModuleFile *, DeclID>::iterator Pos =
-      M.GlobalToLocalDeclIDs.find(Owner);
-  if (Pos == M.GlobalToLocalDeclIDs.end())
+  if (!OrignalModuleFileIndex)
     return LocalDeclID();
 
-  return LocalDeclID(ID - Owner->BaseDeclID + Pos->second);
+  return LocalDeclID(ID, OrignalModuleFileIndex);
 }
 
 GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record,
@@ -7894,32 +7903,34 @@ void ASTReader::FindExternalLexicalDecls(
 
 namespace {
 
-class DeclIDComp {
+class UnalignedDeclIDComp {
   ASTReader &Reader;
   ModuleFile &Mod;
 
 public:
-  DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {}
+  UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M)
+      : Reader(Reader), Mod(M) {}
 
-  bool operator()(LocalDeclID L, LocalDeclID R) const {
+  bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const {
     SourceLocation LHS = getLocation(L);
     SourceLocation RHS = getLocation(R);
     return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
   }
 
-  bool operator()(SourceLocation LHS, LocalDeclID R) const {
+  bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const {
     SourceLocation RHS = getLocation(R);
     return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
   }
 
-  bool operator()(LocalDeclID L, SourceLocation RHS) const {
+  bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const {
     SourceLocation LHS = getLocation(L);
     return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
   }
 
-  SourceLocation getLocation(LocalDeclID ID) const {
+  SourceLocation getLocation(unaligned_decl_id_t ID) const {
     return Reader.getSourceManager().getFileLoc(
-            Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID)));
+        Reader.getSourceLocationForDeclID(
+            Reader.getGlobalDeclID(Mod, (LocalDeclID)ID)));
   }
 };
 
@@ -7942,8 +7953,8 @@ void ASTReader::FindFileRegionDecls(FileID File,
     BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset);
   SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length);
 
-  DeclIDComp DIDComp(*this, *DInfo.Mod);
-  ArrayRef<LocalDeclID>::iterator BeginIt =
+  UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod);
+  ArrayRef<unaligned_decl_id_t>::iterator BeginIt =
       llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp);
   if (BeginIt != DInfo.Decls.begin())
     --BeginIt;
@@ -7952,17 +7963,18 @@ void ASTReader::FindFileRegionDecls(FileID File,
   // to backtrack until we find it otherwise we will fail to report that the
   // region overlaps with an objc container.
   while (BeginIt != DInfo.Decls.begin() &&
-         GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt))
+         GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt)))
              ->isTopLevelDeclInObjCContainer())
     --BeginIt;
 
-  ArrayRef<LocalDeclID>::iterator EndIt =
+  ArrayRef<unaligned_decl_id_t>::iterator EndIt =
       llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp);
   if (EndIt != DInfo.Decls.end())
     ++EndIt;
 
-  for (ArrayRef<LocalDeclID>::iterator DIt = BeginIt; DIt != EndIt; ++DIt)
-    Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt)));
+  for (ArrayRef<unaligned_decl_id_t>::iterator DIt = BeginIt; DIt != EndIt;
+       ++DIt)
+    Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt))));
 }
 
 bool
@@ -8169,7 +8181,6 @@ LLVM_DUMP_METHOD void ASTReader::dump() {
   dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap);
   dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap);
   dumpModuleIDMap("Global type map", GlobalTypeMap);
-  dumpModuleIDMap("Global declaration map", GlobalDeclMap);
   dumpModuleIDMap("Global identifier map", GlobalIdentifierMap);
   dumpModuleIDMap("Global macro map", GlobalMacroMap);
   dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap);
@@ -9185,7 +9196,7 @@ void ASTRecordReader::readUnresolvedSet(LazyASTUnresolvedSet &Set) {
   while (NumDecls--) {
     GlobalDeclID ID = readDeclID();
     AccessSpecifier AS = (AccessSpecifier) readInt();
-    Set.addLazyDecl(getContext(), ID.get(), AS);
+    Set.addLazyDecl(getContext(), ID, AS);
   }
 }
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 765c083ce8df8cbb67adc232a962d906c5692ad5..cf2dc32e30b91e3b55a6857609d661da20c1085d 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D,
   auto *ExistingPattern = Existing->getTemplatedDecl();
   RedeclarableResult Result(
       /*MergeWith*/ ExistingPattern,
-      GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl);
+      DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl);
 
   if (auto *DClass = dyn_cast<CXXRecordDecl>(DPattern)) {
     // Merge with any existing definition.
@@ -3245,11 +3245,10 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) {
 /// Get the correct cursor and offset for loading a declaration.
 ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID,
                                                      SourceLocation &Loc) {
-  GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID);
-  assert(I != GlobalDeclMap.end() && "Corrupted global declaration map");
-  ModuleFile *M = I->second;
-  const DeclOffset &DOffs =
-      M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS];
+  ModuleFile *M = getOwningModuleFile(ID);
+  assert(M);
+  unsigned LocalDeclIndex = ID.getLocalDeclIndex();
+  const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex];
   Loc = ReadSourceLocation(*M, DOffs.getRawLoc());
   return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset));
 }
@@ -3792,7 +3791,6 @@ void ASTReader::markIncompleteDeclChain(Decl *D) {
 
 /// Read the declaration at the given offset from the AST file.
 Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
-  unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS;
   SourceLocation DeclLoc;
   RecordLocation Loc = DeclCursorForID(ID, DeclLoc);
   llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor;
@@ -4123,7 +4121,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
   }
 
   assert(D && "Unknown declaration reading AST file");
-  LoadedDecl(Index, D);
+  LoadedDecl(translateGlobalDeclIDToIndex(ID), D);
   // Set the DeclContext before doing any deserialization, to make sure internal
   // calls to Decl::getASTContext() by Decl's methods will find the
   // TranslationUnitDecl without crashing.
@@ -4449,7 +4447,7 @@ namespace {
                            M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap,
                            Compare);
       if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap ||
-          Result->DefinitionID != LocalID) {
+          Result->getDefinitionID() != LocalID) {
         // We didn't find anything. If the class definition is in this module
         // file, then the module files it depends on cannot have any categories,
         // so suppress further lookup.
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 953990a4aca6db686d6b4ee9887b794f67a2e0de..ee3e687636e6a0451816b4f57999985a65db53cd 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3357,12 +3357,10 @@ void ASTWriter::WriteTypeDeclOffsets() {
   Abbrev = std::make_shared<BitCodeAbbrev>();
   Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block
   unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
   {
-    RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(),
-                                       FirstDeclID.get() - NUM_PREDEF_DECL_IDS};
+    RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()};
     Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets));
   }
 }
@@ -5423,7 +5421,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
                           M.NumPreprocessedEntities);
         writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules);
         writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors);
-        writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls);
         writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes);
       }
     }
@@ -6618,13 +6615,11 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) {
 
   // Note, this will get called multiple times, once one the reader starts up
   // and again each time it's done reading a PCH or module.
-  FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls());
   FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes();
   FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers();
   FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros();
   FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules();
   FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors();
-  NextDeclID = FirstDeclID;
   NextTypeID = FirstTypeID;
   NextIdentID = FirstIdentID;
   NextMacroID = FirstMacroID;
diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp
index 2c42d33a8f5dd3c937dee7b0c198bcc0a29959ba..f64a59bd9489146e0e6535d730232c92177f1d04 100644
--- a/clang/lib/Serialization/ModuleFile.cpp
+++ b/clang/lib/Serialization/ModuleFile.cpp
@@ -87,7 +87,6 @@ LLVM_DUMP_METHOD void ModuleFile::dump() {
                << "  Number of types: " << LocalNumTypes << '\n';
   dumpLocalRemap("Type index local -> global map", TypeRemap);
 
-  llvm::errs() << "  Base decl ID: " << BaseDeclID << '\n'
+  llvm::errs() << "  Base decl index: " << BaseDeclIndex << '\n'
                << "  Number of decls: " << LocalNumDecls << '\n';
-  dumpLocalRemap("Decl ID local -> global map", DeclRemap);
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index cd5a3bdd02e4a614a3341e21cb0523dcaaae3d7e..68e829cace495168a79ac37fab9aebb1dbaaff01 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -78,6 +78,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   NoReturnFunctionChecker.cpp
   NonNullParamChecker.cpp
   NonnullGlobalConstantsChecker.cpp
+  NoOwnershipChangeVisitor.cpp
   NullabilityChecker.cpp
   NumberObjectConversionChecker.cpp
   ObjCAtSyncChecker.cpp
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 467cf8889b6d3f3a277795997ac55db9944c3af5..fe202c79ed62091ae8728bd9fe54d2bb1cdc9918 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -46,6 +46,7 @@
 
 #include "AllocationState.h"
 #include "InterCheckerAPI.h"
+#include "NoOwnershipChangeVisitor.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
@@ -79,13 +80,11 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include <climits>
 #include <functional>
 #include <optional>
 #include <utility>
@@ -414,7 +413,7 @@ private:
   bool isFreeingCall(const CallEvent &Call) const;
   static bool isFreeingOwnershipAttrCall(const FunctionDecl *Func);
 
-  friend class NoOwnershipChangeVisitor;
+  friend class NoMemOwnershipChangeVisitor;
 
   CallDescriptionMap<CheckFn> AllocatingMemFnMap{
       {{CDM::CLibrary, {"alloca"}, 1}, &MallocChecker::checkAlloca},
@@ -765,61 +764,8 @@ private:
 //===----------------------------------------------------------------------===//
 
 namespace {
-class NoOwnershipChangeVisitor final : public NoStateChangeFuncVisitor {
-  // The symbol whose (lack of) ownership change we are interested in.
-  SymbolRef Sym;
-  const MallocChecker &Checker;
-  using OwnerSet = llvm::SmallPtrSet<const MemRegion *, 8>;
-
-  // Collect which entities point to the allocated memory, and could be
-  // responsible for deallocating it.
-  class OwnershipBindingsHandler : public StoreManager::BindingsHandler {
-    SymbolRef Sym;
-    OwnerSet &Owners;
-
-  public:
-    OwnershipBindingsHandler(SymbolRef Sym, OwnerSet &Owners)
-        : Sym(Sym), Owners(Owners) {}
-
-    bool HandleBinding(StoreManager &SMgr, Store Store, const MemRegion *Region,
-                       SVal Val) override {
-      if (Val.getAsSymbol() == Sym)
-        Owners.insert(Region);
-      return true;
-    }
-
-    LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); }
-    LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &out) const {
-      out << "Owners: {\n";
-      for (const MemRegion *Owner : Owners) {
-        out << "  ";
-        Owner->dumpToStream(out);
-        out << ",\n";
-      }
-      out << "}\n";
-    }
-  };
-
+class NoMemOwnershipChangeVisitor final : public NoOwnershipChangeVisitor {
 protected:
-  OwnerSet getOwnersAtNode(const ExplodedNode *N) {
-    OwnerSet Ret;
-
-    ProgramStateRef State = N->getState();
-    OwnershipBindingsHandler Handler{Sym, Ret};
-    State->getStateManager().getStoreManager().iterBindings(State->getStore(),
-                                                            Handler);
-    return Ret;
-  }
-
-  LLVM_DUMP_METHOD static std::string
-  getFunctionName(const ExplodedNode *CallEnterN) {
-    if (const CallExpr *CE = llvm::dyn_cast_or_null<CallExpr>(
-            CallEnterN->getLocationAs<CallEnter>()->getCallExpr()))
-      if (const FunctionDecl *FD = CE->getDirectCallee())
-        return FD->getQualifiedNameAsString();
-    return "";
-  }
-
   /// Syntactically checks whether the callee is a deallocating function. Since
   /// we have no path-sensitive information on this call (we would need a
   /// CallEvent instead of a CallExpr for that), its possible that a
@@ -828,8 +774,9 @@ protected:
   /// See namespace `memory_passed_to_fn_call_free_through_fn_ptr` in
   /// clang/test/Analysis/NewDeleteLeaks.cpp.
   bool isFreeingCallAsWritten(const CallExpr &Call) const {
-    if (Checker.FreeingMemFnMap.lookupAsWritten(Call) ||
-        Checker.ReallocatingMemFnMap.lookupAsWritten(Call))
+    const auto *MallocChk = static_cast<const MallocChecker *>(&Checker);
+    if (MallocChk->FreeingMemFnMap.lookupAsWritten(Call) ||
+        MallocChk->ReallocatingMemFnMap.lookupAsWritten(Call))
       return true;
 
     if (const auto *Func =
@@ -839,23 +786,21 @@ protected:
     return false;
   }
 
+  bool hasResourceStateChanged(ProgramStateRef CallEnterState,
+                               ProgramStateRef CallExitEndState) final {
+    return CallEnterState->get<RegionState>(Sym) !=
+           CallExitEndState->get<RegionState>(Sym);
+  }
+
   /// Heuristically guess whether the callee intended to free memory. This is
   /// done syntactically, because we are trying to argue about alternative
   /// paths of execution, and as a consequence we don't have path-sensitive
   /// information.
-  bool doesFnIntendToHandleOwnership(const Decl *Callee, ASTContext &ACtx) {
+  bool doesFnIntendToHandleOwnership(const Decl *Callee,
+                                     ASTContext &ACtx) final {
     using namespace clang::ast_matchers;
     const FunctionDecl *FD = dyn_cast<FunctionDecl>(Callee);
 
-    // Given that the stack frame was entered, the body should always be
-    // theoretically obtainable. In case of body farms, the synthesized body
-    // is not attached to declaration, thus triggering the '!FD->hasBody()'
-    // branch. That said, would a synthesized body ever intend to handle
-    // ownership? As of today they don't. And if they did, how would we
-    // put notes inside it, given that it doesn't match any source locations?
-    if (!FD || !FD->hasBody())
-      return false;
-
     auto Matches = match(findAll(stmt(anyOf(cxxDeleteExpr().bind("delete"),
                                             callExpr().bind("call")))),
                          *FD->getBody(), ACtx);
@@ -873,30 +818,7 @@ protected:
     return false;
   }
 
-  bool wasModifiedInFunction(const ExplodedNode *CallEnterN,
-                             const ExplodedNode *CallExitEndN) override {
-    if (!doesFnIntendToHandleOwnership(
-            CallExitEndN->getFirstPred()->getLocationContext()->getDecl(),
-            CallExitEndN->getState()->getAnalysisManager().getASTContext()))
-      return true;
-
-    if (CallEnterN->getState()->get<RegionState>(Sym) !=
-        CallExitEndN->getState()->get<RegionState>(Sym))
-      return true;
-
-    OwnerSet CurrOwners = getOwnersAtNode(CallEnterN);
-    OwnerSet ExitOwners = getOwnersAtNode(CallExitEndN);
-
-    // Owners in the current set may be purged from the analyzer later on.
-    // If a variable is dead (is not referenced directly or indirectly after
-    // some point), it will be removed from the Store before the end of its
-    // actual lifetime.
-    // This means that if the ownership status didn't change, CurrOwners
-    // must be a superset of, but not necessarily equal to ExitOwners.
-    return !llvm::set_is_subset(ExitOwners, CurrOwners);
-  }
-
-  static PathDiagnosticPieceRef emitNote(const ExplodedNode *N) {
+  PathDiagnosticPieceRef emitNote(const ExplodedNode *N) final {
     PathDiagnosticLocation L = PathDiagnosticLocation::create(
         N->getLocation(),
         N->getState()->getStateManager().getContext().getSourceManager());
@@ -905,42 +827,9 @@ protected:
            "later deallocation");
   }
 
-  PathDiagnosticPieceRef
-  maybeEmitNoteForObjCSelf(PathSensitiveBugReport &R,
-                           const ObjCMethodCall &Call,
-                           const ExplodedNode *N) override {
-    // TODO: Implement.
-    return nullptr;
-  }
-
-  PathDiagnosticPieceRef
-  maybeEmitNoteForCXXThis(PathSensitiveBugReport &R,
-                          const CXXConstructorCall &Call,
-                          const ExplodedNode *N) override {
-    // TODO: Implement.
-    return nullptr;
-  }
-
-  PathDiagnosticPieceRef
-  maybeEmitNoteForParameters(PathSensitiveBugReport &R, const CallEvent &Call,
-                             const ExplodedNode *N) override {
-    // TODO: Factor the logic of "what constitutes as an entity being passed
-    // into a function call" out by reusing the code in
-    // NoStoreFuncVisitor::maybeEmitNoteForParameters, maybe by incorporating
-    // the printing technology in UninitializedObject's FieldChainInfo.
-    ArrayRef<ParmVarDecl *> Parameters = Call.parameters();
-    for (unsigned I = 0; I < Call.getNumArgs() && I < Parameters.size(); ++I) {
-      SVal V = Call.getArgSVal(I);
-      if (V.getAsSymbol() == Sym)
-        return emitNote(N);
-    }
-    return nullptr;
-  }
-
 public:
-  NoOwnershipChangeVisitor(SymbolRef Sym, const MallocChecker *Checker)
-      : NoStateChangeFuncVisitor(bugreporter::TrackingKind::Thorough), Sym(Sym),
-        Checker(*Checker) {}
+  NoMemOwnershipChangeVisitor(SymbolRef Sym, const MallocChecker *Checker)
+      : NoOwnershipChangeVisitor(Sym, Checker) {}
 
   void Profile(llvm::FoldingSetNodeID &ID) const override {
     static int Tag = 0;
@@ -2949,7 +2838,7 @@ void MallocChecker::HandleLeak(SymbolRef Sym, ExplodedNode *N,
   R->markInteresting(Sym);
   R->addVisitor<MallocBugVisitor>(Sym, true);
   if (ShouldRegisterNoOwnershipChangeVisitor)
-    R->addVisitor<NoOwnershipChangeVisitor>(Sym, this);
+    R->addVisitor<NoMemOwnershipChangeVisitor>(Sym, this);
   C.emitReport(std::move(R));
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2ff76679b5ebf744d93a832a7f1c0763521cbab9
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp
@@ -0,0 +1,116 @@
+//===--------------------------------------------------------------*- C++ -*--//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NoOwnershipChangeVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h"
+#include "llvm/ADT/SetOperations.h"
+
+using namespace clang;
+using namespace ento;
+using OwnerSet = NoOwnershipChangeVisitor::OwnerSet;
+
+// Collect which entities point to the allocated memory, and could be
+// responsible for deallocating it.
+class OwnershipBindingsHandler : public StoreManager::BindingsHandler {
+  SymbolRef Sym;
+  OwnerSet &Owners;
+
+public:
+  OwnershipBindingsHandler(SymbolRef Sym, OwnerSet &Owners)
+      : Sym(Sym), Owners(Owners) {}
+
+  bool HandleBinding(StoreManager &SMgr, Store Store, const MemRegion *Region,
+                     SVal Val) override {
+    if (Val.getAsSymbol() == Sym)
+      Owners.insert(Region);
+    return true;
+  }
+
+  LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); }
+  LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &out) const {
+    out << "Owners: {\n";
+    for (const MemRegion *Owner : Owners) {
+      out << "  ";
+      Owner->dumpToStream(out);
+      out << ",\n";
+    }
+    out << "}\n";
+  }
+};
+
+OwnerSet NoOwnershipChangeVisitor::getOwnersAtNode(const ExplodedNode *N) {
+  OwnerSet Ret;
+
+  ProgramStateRef State = N->getState();
+  OwnershipBindingsHandler Handler{Sym, Ret};
+  State->getStateManager().getStoreManager().iterBindings(State->getStore(),
+                                                          Handler);
+  return Ret;
+}
+
+LLVM_DUMP_METHOD std::string
+NoOwnershipChangeVisitor::getFunctionName(const ExplodedNode *CallEnterN) {
+  if (const CallExpr *CE = llvm::dyn_cast_or_null<CallExpr>(
+          CallEnterN->getLocationAs<CallEnter>()->getCallExpr()))
+    if (const FunctionDecl *FD = CE->getDirectCallee())
+      return FD->getQualifiedNameAsString();
+  return "";
+}
+
+bool NoOwnershipChangeVisitor::wasModifiedInFunction(
+    const ExplodedNode *CallEnterN, const ExplodedNode *CallExitEndN) {
+  const Decl *Callee =
+      CallExitEndN->getFirstPred()->getLocationContext()->getDecl();
+  const FunctionDecl *FD = dyn_cast<FunctionDecl>(Callee);
+
+  // Given that the stack frame was entered, the body should always be
+  // theoretically obtainable. In case of body farms, the synthesized body
+  // is not attached to declaration, thus triggering the '!FD->hasBody()'
+  // branch. That said, would a synthesized body ever intend to handle
+  // ownership? As of today they don't. And if they did, how would we
+  // put notes inside it, given that it doesn't match any source locations?
+  if (!FD || !FD->hasBody())
+    return false;
+  if (!doesFnIntendToHandleOwnership(
+          Callee,
+          CallExitEndN->getState()->getAnalysisManager().getASTContext()))
+    return true;
+
+  if (hasResourceStateChanged(CallEnterN->getState(), CallExitEndN->getState()))
+    return true;
+
+  OwnerSet CurrOwners = getOwnersAtNode(CallEnterN);
+  OwnerSet ExitOwners = getOwnersAtNode(CallExitEndN);
+
+  // Owners in the current set may be purged from the analyzer later on.
+  // If a variable is dead (is not referenced directly or indirectly after
+  // some point), it will be removed from the Store before the end of its
+  // actual lifetime.
+  // This means that if the ownership status didn't change, CurrOwners
+  // must be a superset of, but not necessarily equal to ExitOwners.
+  return !llvm::set_is_subset(ExitOwners, CurrOwners);
+}
+
+PathDiagnosticPieceRef NoOwnershipChangeVisitor::maybeEmitNoteForParameters(
+    PathSensitiveBugReport &R, const CallEvent &Call, const ExplodedNode *N) {
+  // TODO: Factor the logic of "what constitutes as an entity being passed
+  // into a function call" out by reusing the code in
+  // NoStoreFuncVisitor::maybeEmitNoteForParameters, maybe by incorporating
+  // the printing technology in UninitializedObject's FieldChainInfo.
+  ArrayRef<ParmVarDecl *> Parameters = Call.parameters();
+  for (unsigned I = 0; I < Call.getNumArgs() && I < Parameters.size(); ++I) {
+    SVal V = Call.getArgSVal(I);
+    if (V.getAsSymbol() == Sym)
+      return emitNote(N);
+  }
+  return nullptr;
+}
diff --git a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h
new file mode 100644
index 0000000000000000000000000000000000000000..027f1a156a7c03d2fdc3b9ecda1be3d64a7e9b67
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h
@@ -0,0 +1,77 @@
+//===--------------------------------------------------------------*- C++ -*--//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h"
+
+namespace clang {
+namespace ento {
+
+class NoOwnershipChangeVisitor : public NoStateChangeFuncVisitor {
+protected:
+  // The symbol whose (lack of) ownership change we are interested in.
+  SymbolRef Sym;
+  const CheckerBase &Checker;
+
+  LLVM_DUMP_METHOD static std::string
+  getFunctionName(const ExplodedNode *CallEnterN);
+
+  /// Heuristically guess whether the callee intended to free the resource. This
+  /// is done syntactically, because we are trying to argue about alternative
+  /// paths of execution, and as a consequence we don't have path-sensitive
+  /// information.
+  virtual bool doesFnIntendToHandleOwnership(const Decl *Callee,
+                                             ASTContext &ACtx) = 0;
+
+  virtual bool hasResourceStateChanged(ProgramStateRef CallEnterState,
+                                       ProgramStateRef CallExitEndState) = 0;
+
+  bool wasModifiedInFunction(const ExplodedNode *CallEnterN,
+                             const ExplodedNode *CallExitEndN) final;
+
+  virtual PathDiagnosticPieceRef emitNote(const ExplodedNode *N) = 0;
+
+  PathDiagnosticPieceRef maybeEmitNoteForObjCSelf(PathSensitiveBugReport &R,
+                                                  const ObjCMethodCall &Call,
+                                                  const ExplodedNode *N) final {
+    // TODO: Implement.
+    return nullptr;
+  }
+
+  PathDiagnosticPieceRef maybeEmitNoteForCXXThis(PathSensitiveBugReport &R,
+                                                 const CXXConstructorCall &Call,
+                                                 const ExplodedNode *N) final {
+    // TODO: Implement.
+    return nullptr;
+  }
+
+  // Set this to final, effectively dispatch to emitNote.
+  PathDiagnosticPieceRef
+  maybeEmitNoteForParameters(PathSensitiveBugReport &R, const CallEvent &Call,
+                             const ExplodedNode *N) final;
+
+public:
+  using OwnerSet = llvm::SmallPtrSet<const MemRegion *, 8>;
+
+private:
+  OwnerSet getOwnersAtNode(const ExplodedNode *N);
+
+public:
+  NoOwnershipChangeVisitor(SymbolRef Sym, const CheckerBase *Checker)
+      : NoStateChangeFuncVisitor(bugreporter::TrackingKind::Thorough), Sym(Sym),
+        Checker(*Checker) {}
+
+  void Profile(llvm::FoldingSetNodeID &ID) const override {
+    static int Tag = 0;
+    ID.AddPointer(&Tag);
+    ID.AddPointer(Sym);
+  }
+};
+} // namespace ento
+} // namespace clang
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index 2438cf30b39b5944a02af256990d3a7907c309e5..b73534136fdf02019403057c3827b09211da2b7f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -17,6 +17,7 @@
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h"
 #include "llvm/ADT/StringRef.h"
 
 using namespace clang;
@@ -26,16 +27,88 @@ namespace {
 class PointerSubChecker
   : public Checker< check::PreStmt<BinaryOperator> > {
   const BugType BT{this, "Pointer subtraction"};
+  const llvm::StringLiteral Msg_MemRegionDifferent =
+      "Subtraction of two pointers that do not point into the same array "
+      "is undefined behavior.";
+  const llvm::StringLiteral Msg_LargeArrayIndex =
+      "Using an array index greater than the array size at pointer subtraction "
+      "is undefined behavior.";
+  const llvm::StringLiteral Msg_NegativeArrayIndex =
+      "Using a negative array index at pointer subtraction "
+      "is undefined behavior.";
+  const llvm::StringLiteral Msg_BadVarIndex =
+      "Indexing the address of a variable with other than 1 at this place "
+      "is undefined behavior.";
+
+  bool checkArrayBounds(CheckerContext &C, const Expr *E,
+                        const ElementRegion *ElemReg,
+                        const MemRegion *Reg) const;
+  void reportBug(CheckerContext &C, const Expr *E,
+                 const llvm::StringLiteral &Msg) const;
 
 public:
   void checkPreStmt(const BinaryOperator *B, CheckerContext &C) const;
 };
 }
 
+bool PointerSubChecker::checkArrayBounds(CheckerContext &C, const Expr *E,
+                                         const ElementRegion *ElemReg,
+                                         const MemRegion *Reg) const {
+  if (!ElemReg)
+    return true;
+
+  ProgramStateRef State = C.getState();
+  const MemRegion *SuperReg = ElemReg->getSuperRegion();
+  SValBuilder &SVB = C.getSValBuilder();
+
+  if (SuperReg == Reg) {
+    if (const llvm::APSInt *I = SVB.getKnownValue(State, ElemReg->getIndex());
+        I && (!I->isOne() && !I->isZero()))
+      reportBug(C, E, Msg_BadVarIndex);
+    return false;
+  }
+
+  DefinedOrUnknownSVal ElemCount =
+      getDynamicElementCount(State, SuperReg, SVB, ElemReg->getElementType());
+  auto IndexTooLarge = SVB.evalBinOp(C.getState(), BO_GT, ElemReg->getIndex(),
+                                     ElemCount, SVB.getConditionType())
+                           .getAs<DefinedOrUnknownSVal>();
+  if (IndexTooLarge) {
+    ProgramStateRef S1, S2;
+    std::tie(S1, S2) = C.getState()->assume(*IndexTooLarge);
+    if (S1 && !S2) {
+      reportBug(C, E, Msg_LargeArrayIndex);
+      return false;
+    }
+  }
+  auto IndexTooSmall = SVB.evalBinOp(State, BO_LT, ElemReg->getIndex(),
+                                     SVB.makeZeroVal(SVB.getArrayIndexType()),
+                                     SVB.getConditionType())
+                           .getAs<DefinedOrUnknownSVal>();
+  if (IndexTooSmall) {
+    ProgramStateRef S1, S2;
+    std::tie(S1, S2) = State->assume(*IndexTooSmall);
+    if (S1 && !S2) {
+      reportBug(C, E, Msg_NegativeArrayIndex);
+      return false;
+    }
+  }
+  return true;
+}
+
+void PointerSubChecker::reportBug(CheckerContext &C, const Expr *E,
+                                  const llvm::StringLiteral &Msg) const {
+  if (ExplodedNode *N = C.generateNonFatalErrorNode()) {
+    auto R = std::make_unique<PathSensitiveBugReport>(BT, Msg, N);
+    R->addRange(E->getSourceRange());
+    C.emitReport(std::move(R));
+  }
+}
+
 void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
                                      CheckerContext &C) const {
   // When doing pointer subtraction, if the two pointers do not point to the
-  // same memory chunk, emit a warning.
+  // same array, emit a warning.
   if (B->getOpcode() != BO_Sub)
     return;
 
@@ -44,28 +117,36 @@ void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
 
   const MemRegion *LR = LV.getAsRegion();
   const MemRegion *RR = RV.getAsRegion();
-
-  if (!(LR && RR))
+  if (!LR || !RR)
     return;
 
-  const MemRegion *BaseLR = LR->getBaseRegion();
-  const MemRegion *BaseRR = RR->getBaseRegion();
+  // Allow subtraction of identical pointers.
+  if (LR == RR)
+    return;
 
-  if (BaseLR == BaseRR)
+  // No warning if one operand is unknown.
+  if (isa<SymbolicRegion>(LR) || isa<SymbolicRegion>(RR))
     return;
 
-  // Allow arithmetic on different symbolic regions.
-  if (isa<SymbolicRegion>(BaseLR) || isa<SymbolicRegion>(BaseRR))
+  const auto *ElemLR = dyn_cast<ElementRegion>(LR);
+  const auto *ElemRR = dyn_cast<ElementRegion>(RR);
+
+  if (!checkArrayBounds(C, B->getLHS(), ElemLR, RR))
+    return;
+  if (!checkArrayBounds(C, B->getRHS(), ElemRR, LR))
     return;
 
-  if (ExplodedNode *N = C.generateNonFatalErrorNode()) {
-    constexpr llvm::StringLiteral Msg =
-        "Subtraction of two pointers that do not point to the same memory "
-        "chunk may cause incorrect result.";
-    auto R = std::make_unique<PathSensitiveBugReport>(BT, Msg, N);
-    R->addRange(B->getSourceRange());
-    C.emitReport(std::move(R));
+  if (ElemLR && ElemRR) {
+    const MemRegion *SuperLR = ElemLR->getSuperRegion();
+    const MemRegion *SuperRR = ElemRR->getSuperRegion();
+    if (SuperLR == SuperRR)
+      return;
+    // Allow arithmetic on different symbolic regions.
+    if (isa<SymbolicRegion>(SuperLR) || isa<SymbolicRegion>(SuperRR))
+      return;
   }
+
+  reportBug(C, B, Msg_MemRegionDifferent);
 }
 
 void ento::registerPointerSubChecker(CheckerManager &mgr) {
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index 290d96611d466430270e3a30ac3f9a7c1893a6c2..197d67310728519af0fb3924abb1bb2eaff32789 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1971,45 +1971,33 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       ExplodedNodeSet Tmp;
       StmtNodeBuilder Bldr2(PreVisit, Tmp, *currBldrCtx);
 
-      bool HasRewrittenInit = false;
-      const Expr *ArgE = nullptr;
-      if (const auto *DefE = dyn_cast<CXXDefaultArgExpr>(S)) {
+      const Expr *ArgE;
+      if (const auto *DefE = dyn_cast<CXXDefaultArgExpr>(S))
         ArgE = DefE->getExpr();
-        HasRewrittenInit = DefE->hasRewrittenInit();
-      } else if (const auto *DefE = dyn_cast<CXXDefaultInitExpr>(S)) {
+      else if (const auto *DefE = dyn_cast<CXXDefaultInitExpr>(S))
         ArgE = DefE->getExpr();
-        HasRewrittenInit = DefE->hasRewrittenInit();
-      } else
+      else
         llvm_unreachable("unknown constant wrapper kind");
 
-      if (HasRewrittenInit) {
-        for (auto *N : PreVisit) {
-          ProgramStateRef state = N->getState();
-          const LocationContext *LCtx = N->getLocationContext();
-          state = state->BindExpr(S, LCtx, state->getSVal(ArgE, LCtx));
-          Bldr2.generateNode(S, N, state);
-        }
-      } else {
-        // If it's not rewritten, the contents of these expressions are not
-        // actually part of the current function, so we fall back to constant
-        // evaluation.
-        bool IsTemporary = false;
-        if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(ArgE)) {
-          ArgE = MTE->getSubExpr();
-          IsTemporary = true;
-        }
-
-        std::optional<SVal> ConstantVal = svalBuilder.getConstantVal(ArgE);
-        const LocationContext *LCtx = Pred->getLocationContext();
-        for (auto *I : PreVisit) {
-          ProgramStateRef State = I->getState();
-          State = State->BindExpr(S, LCtx, ConstantVal.value_or(UnknownVal()));
-          if (IsTemporary)
-            State = createTemporaryRegionIfNeeded(State, LCtx, cast<Expr>(S),
-                                                  cast<Expr>(S));
+      bool IsTemporary = false;
+      if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(ArgE)) {
+        ArgE = MTE->getSubExpr();
+        IsTemporary = true;
+      }
 
-          Bldr2.generateNode(S, I, State);
-        }
+      std::optional<SVal> ConstantVal = svalBuilder.getConstantVal(ArgE);
+      if (!ConstantVal)
+        ConstantVal = UnknownVal();
+
+      const LocationContext *LCtx = Pred->getLocationContext();
+      for (const auto I : PreVisit) {
+        ProgramStateRef State = I->getState();
+        State = State->BindExpr(S, LCtx, *ConstantVal);
+        if (IsTemporary)
+          State = createTemporaryRegionIfNeeded(State, LCtx,
+                                                cast<Expr>(S),
+                                                cast<Expr>(S));
+        Bldr2.generateNode(S, I, State);
       }
 
       getCheckerManager().runCheckersForPostStmt(Dst, Tmp, S, *this);
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 66a2f6e0acb63828ad36b116170a310f1d8aabc0..0cab17a34244069f1d1f380e42651eee67755334 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -42,7 +42,7 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
 }
 
 bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated(
-    EntryRef Ref, const LangOptions &LangOpts) {
+    EntryRef Ref) {
   auto &Entry = Ref.Entry;
 
   if (Entry.isError() || Entry.isDirectory())
@@ -66,7 +66,7 @@ bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated(
   // dependencies.
   if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(),
                                         Contents->DepDirectiveTokens,
-                                        Directives, LangOpts)) {
+                                        Directives)) {
     Contents->DepDirectiveTokens.clear();
     // FIXME: Propagate the diagnostic if desired by the client.
     Contents->DepDirectives.store(new std::optional<DependencyDirectivesTy>());
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 07e1960dd90583a7b081c190b2b61acce96347e2..0f82f22d8b9a8c7d1dd360ae39d668565bfec7b3 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -364,12 +364,11 @@ public:
     // Use the dependency scanning optimized file system if requested to do so.
     if (DepFS)
       ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
-          [LocalDepFS = DepFS,
-           &LangOpts = ScanInstance.getLangOpts()](FileEntryRef File)
+          [LocalDepFS = DepFS](FileEntryRef File)
           -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
         if (llvm::ErrorOr<EntryRef> Entry =
                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
-          if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry, LangOpts))
+          if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry))
             return Entry->getDirectiveTokens();
         return std::nullopt;
       };
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 8d32c45a4a70cf55138ab82be6345171c8e2cd99..0a656dff384217f3975a395a465000211f673257 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -383,12 +383,13 @@ llvm::ArrayRef<syntax::Token> TokenBuffer::spelledTokens(FileID FID) const {
   return It->second.SpelledTokens;
 }
 
-const syntax::Token *TokenBuffer::spelledTokenAt(SourceLocation Loc) const {
+const syntax::Token *
+TokenBuffer::spelledTokenContaining(SourceLocation Loc) const {
   assert(Loc.isFileID());
   const auto *Tok = llvm::partition_point(
       spelledTokens(SourceMgr->getFileID(Loc)),
-      [&](const syntax::Token &Tok) { return Tok.location() < Loc; });
-  if (!Tok || Tok->location() != Loc)
+      [&](const syntax::Token &Tok) { return Tok.endLocation() <= Loc; });
+  if (!Tok || Loc < Tok->location())
     return nullptr;
   return Tok;
 }
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index dd5064d993e667b72664c6ee29ef88bde444d7dd..6f6fca8c1cfd8eb27e3897ca33a3d4e7c0eec7df 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -609,3 +609,17 @@ namespace ArrayMemberAccess {
     bool cond = a->x;
   }
 }
+
+namespace OnePastEndSub {
+  struct A {};
+  constexpr A a[3][3];
+  constexpr int diff2 = &a[1][3] - &a[1][0]; /// Used to crash.
+}
+
+static int same_entity_2[3];
+constexpr int *get2() {
+  // This is a redeclaration of the same entity, even though it doesn't
+  // inherit the type of the prior declaration.
+  extern int same_entity_2[];
+  return same_entity_2;
+}
diff --git a/clang/test/AST/Interp/builtin-align-cxx.cpp b/clang/test/AST/Interp/builtin-align-cxx.cpp
index c4103953df026a15a98498143f6a964e75646aa9..a1edf307d6c479c57588237ee2934593d41b04bc 100644
--- a/clang/test/AST/Interp/builtin-align-cxx.cpp
+++ b/clang/test/AST/Interp/builtin-align-cxx.cpp
@@ -202,8 +202,7 @@ static_assert(__builtin_align_down(&align32array[7], 4) == &align32array[4], "")
 static_assert(__builtin_align_down(&align32array[8], 4) == &align32array[8], "");
 
 // Achieving the same thing using casts to uintptr_t is not allowed:
-static_assert((char *)((__UINTPTR_TYPE__)&align32array[7] & ~3) == &align32array[4], ""); // both-error{{not an integral constant expression}} \
-                                                                                          // expected-note {{cast that performs the conversions of a reinterpret_cast is not allowed in a constant expression}}
+static_assert((char *)((__UINTPTR_TYPE__)&align32array[7] & ~3) == &align32array[4], ""); // both-error{{not an integral constant expression}}
 
 static_assert(__builtin_align_down(&align32array[1], 4) == &align32array[0], "");
 static_assert(__builtin_align_down(&align32array[1], 64) == &align32array[0], ""); // both-error{{not an integral constant expression}}
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c
index f4c7bf16f2f95ee8ae44392fcdf82ac9c3842f8f..1cc450e48def06f9fadc23ef24a70f6b56c9bc24 100644
--- a/clang/test/AST/Interp/c.c
+++ b/clang/test/AST/Interp/c.c
@@ -66,12 +66,10 @@ _Static_assert((&a - 100) != 0, ""); // pedantic-ref-warning {{is a GNU extensio
                                      // pedantic-ref-note {{-100 of non-array}} \
                                      // pedantic-expected-note {{-100 of non-array}}
 /// extern variable of a composite type.
-/// FIXME: The 'this conversion is not allowed' note is missing in the new interpreter.
 extern struct Test50S Test50;
 _Static_assert(&Test50 != (void*)0, ""); // all-warning {{always true}} \
-                                         // pedantic-ref-warning {{is a GNU extension}} \
-                                         // pedantic-ref-note {{this conversion is not allowed in a constant expression}} \
-                                         // pedantic-expected-warning {{is a GNU extension}}
+                                         // pedantic-warning {{is a GNU extension}} \
+                                         // pedantic-note {{this conversion is not allowed in a constant expression}}
 
 struct y {int x,y;};
 int a2[(intptr_t)&((struct y*)0)->y]; // all-warning {{folded to constant array}}
diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/Interp/complex.cpp
index 09cb620d7b7c398cf1e51312a2ff64af3adc48cf..003f33e092d25feb372860ac0d574a7122f394ac 100644
--- a/clang/test/AST/Interp/complex.cpp
+++ b/clang/test/AST/Interp/complex.cpp
@@ -115,6 +115,11 @@ static_assert(__imag(Doubles[2]) == 0.0, "");
 static_assert(__real(Doubles[3]) == 0.0, "");
 static_assert(__imag(Doubles[3]) == 0.0, "");
 
+static_assert(~(0.5 + 1.5j) == (0.5 + -1.5j), "");
+
+static_assert(__extension__ __imag(A) == 0, "");
+static_assert(__imag(__extension__ A) == 0, "");
+
 void func(void) {
   __complex__ int arr;
   _Complex int result;
diff --git a/clang/test/AST/Interp/const-eval.c b/clang/test/AST/Interp/const-eval.c
index 72c0833a0f6309bd2523017c979d1bbecedfe7fc..eab14c08ec809acc18f02ab42cd57a930329362a 100644
--- a/clang/test/AST/Interp/const-eval.c
+++ b/clang/test/AST/Interp/const-eval.c
@@ -140,15 +140,8 @@ EVAL_EXPR(47, &x < &x + 1 ? 1 : -1)
 EVAL_EXPR(48, &x != &x - 1 ? 1 : -1)
 EVAL_EXPR(49, &x < &x - 100 ? 1 : -1) // ref-error {{not an integer constant expression}}
 
-/// FIXME: Rejecting this is correct, BUT when converting the innermost pointer
-/// to an integer, we do not preserve the information where it came from. So when we later
-/// create a pointer from it, it also doesn't have that information, which means
-/// hasSameBase() for those two pointers will return false. And in those cases, we emit
-/// the diagnostic:
-///   comparison between '&Test50' and '&(631578)' has unspecified value
 extern struct Test50S Test50;
-EVAL_EXPR(50, &Test50 < (struct Test50S*)((unsigned long)&Test50 + 10)) // both-error {{not an integer constant expression}} \
-                                                                        // expected-note {{comparison between}}
+EVAL_EXPR(50, &Test50 < (struct Test50S*)((unsigned long)&Test50 + 10)) // both-error {{not an integer constant expression}}
 
 EVAL_EXPR(51, 0 != (float)1e99)
 
diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/Interp/cxx20.cpp
index 000ffe39eb94a73caca71f5704ad6bbce164c1ca..434823644a7a38980d5dd39c438789e2b11572fb 100644
--- a/clang/test/AST/Interp/cxx20.cpp
+++ b/clang/test/AST/Interp/cxx20.cpp
@@ -774,3 +774,43 @@ void overflowInSwitchCase(int n) {
     break;
   }
 }
+
+namespace APValues {
+  int g;
+  struct A { union { int n, m; }; int *p; int A::*q; char buffer[32]; };
+  template<A a> constexpr const A &get = a;
+  constexpr const A &v = get<A{}>;
+  constexpr const A &w = get<A{1, &g, &A::n, "hello"}>;
+}
+
+namespace self_referencing {
+  struct S {
+    S* ptr = nullptr;
+    constexpr S(int i) : ptr(this) {
+      if (this == ptr && i)
+        ptr = nullptr;
+    }
+    constexpr ~S() {}
+  };
+
+  void test() {
+    S s(1);
+  }
+}
+
+namespace GH64949 {
+  struct f {
+    int g; // both-note {{subobject declared here}}
+    constexpr ~f() {}
+  };
+
+  class h {
+  public:
+    consteval h(char *) {}
+    f i;
+  };
+
+  void test() { h{nullptr}; } // both-error {{call to consteval function 'GH64949::h::h' is not a constant expression}} \
+                              // both-note {{subobject 'g' is not initialized}} \
+                              // both-warning {{expression result unused}}
+}
diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp
index c91d52c552b12792f240cb78def8521e566c6338..1efd784abbbe8fbe139aeba16f33a33a80f10a03 100644
--- a/clang/test/AST/Interp/cxx23.cpp
+++ b/clang/test/AST/Interp/cxx23.cpp
@@ -178,3 +178,25 @@ namespace ExplicitLambdaThis {
   };
   static_assert(f());
 }
+
+namespace std {
+  struct strong_ordering {
+    int n;
+    constexpr operator int() const { return n; }
+    static const strong_ordering less, equal, greater;
+  };
+  constexpr strong_ordering strong_ordering::less = {-1};
+  constexpr strong_ordering strong_ordering::equal = {0};
+  constexpr strong_ordering strong_ordering::greater = {1};
+}
+
+namespace UndefinedThreeWay {
+  struct A {
+    friend constexpr std::strong_ordering operator<=>(const A&, const A&) = default; // all-note {{declared here}}
+  };
+
+  constexpr std::strong_ordering operator<=>(const A&, const A&) noexcept;
+  constexpr std::strong_ordering (*test_a_threeway)(const A&, const A&) = &operator<=>;
+  static_assert(!(*test_a_threeway)(A(), A())); // all-error {{static assertion expression is not an integral constant expression}} \
+                                                // all-note {{undefined function 'operator<=>' cannot be used in a constant expression}}
+}
diff --git a/clang/test/AST/Interp/eval-order.cpp b/clang/test/AST/Interp/eval-order.cpp
index aaf2b74510bbfedd396a7a74fc90d42d0eb6da18..7a7ce6a714601ac1bb0aba5d91ec2bf233d1dc01 100644
--- a/clang/test/AST/Interp/eval-order.cpp
+++ b/clang/test/AST/Interp/eval-order.cpp
@@ -71,8 +71,8 @@ namespace EvalOrder {
   // Rules 1 and 2 have no effect ('b' is not an expression).
 
   // Rule 3: a->*b
-  // SEQ(A(ud).*B(&UserDefined::n)); FIXME
-  // SEQ(A(&ud)->*B(&UserDefined::n)); FIXME
+  SEQ(A(ud).*B(&UserDefined::n));
+  SEQ(A(&ud)->*B(&UserDefined::n));
 
   // Rule 4: a(b1, b2, b3)
   SEQ(A(f)(B(1), B(2), B(3))); // expected-error {{not an integral constant expression}} FIXME \
diff --git a/clang/test/AST/Interp/lambda.cpp b/clang/test/AST/Interp/lambda.cpp
index 77e035ce2547036a3f0488669506aa421fc03918..0eb12643b1b7f425be2cd9d028fbd37553eff7de 100644
--- a/clang/test/AST/Interp/lambda.cpp
+++ b/clang/test/AST/Interp/lambda.cpp
@@ -264,3 +264,19 @@ namespace CaptureDefaults {
   };
   static_assert(f2() == 3, "");
 }
+
+constexpr auto t4 = ([x=42]() consteval { return x; }());
+static_assert(t4 == 42, "");
+
+namespace InvalidCapture {
+
+  int &f(int *p);
+  char &f(...);
+  void g() {
+    int n = -1;   // both-note {{declared here}}
+    [=] {
+      int arr[n]; // both-warning {{variable length arrays in C++ are a Clang extension}} \
+                     both-note {{read of non-const variable 'n' is not allowed in a constant expression}}
+    } ();
+  }
+}
diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp
index c160be06dd241cad80004813e87aa3bc729dc82e..5a29013a053a20bbec7b83f65172ba8bba5f6dab 100644
--- a/clang/test/AST/Interp/literals.cpp
+++ b/clang/test/AST/Interp/literals.cpp
@@ -66,7 +66,12 @@ namespace ScalarTypes {
     First = 0,
   };
   static_assert(getScalar<E>() == First, "");
-  /// FIXME: Member pointers.
+
+  struct S {
+    int v;
+  };
+  constexpr int S::* MemberPtr = &S::v;
+  static_assert(getScalar<decltype(MemberPtr)>() == nullptr, "");
 
 #if __cplusplus >= 201402L
   constexpr void Void(int n) {
@@ -1204,7 +1209,7 @@ namespace incdecbool {
 constexpr int externvar1() { // both-error {{never produces a constant expression}}
   extern char arr[]; // ref-note {{declared here}}
    return arr[0]; // ref-note {{read of non-constexpr variable 'arr'}} \
-                  // expected-note {{array-to-pointer decay of array member without known bound is not supported}}
+                  // expected-note {{indexing of array without known bound}}
 }
 #endif
 
diff --git a/clang/test/AST/Interp/memberpointers.cpp b/clang/test/AST/Interp/memberpointers.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..54d73fe86ca18d5781eb2595edd130e1b3b49e47
--- /dev/null
+++ b/clang/test/AST/Interp/memberpointers.cpp
@@ -0,0 +1,197 @@
+// RUN: %clang_cc1 -std=c++14 -fexperimental-new-constant-interpreter -verify=expected,both %s
+// RUN: %clang_cc1 -std=c++14 -verify=ref,both %s
+
+namespace MemberPointers {
+  struct A {
+    constexpr A(int n) : n(n) {}
+    int n;
+    constexpr int f() const { return n + 3; }
+  };
+
+  constexpr A a(7);
+  static_assert(A(5).*&A::n == 5, "");
+  static_assert((&a)->*&A::n == 7, "");
+  static_assert((A(8).*&A::f)() == 11, "");
+  static_assert(((&a)->*&A::f)() == 10, "");
+
+  struct B : A {
+    constexpr B(int n, int m) : A(n), m(m) {}
+    int m;
+    constexpr int g() const { return n + m + 1; }
+  };
+  constexpr B b(9, 13);
+  static_assert(B(4, 11).*&A::n == 4, "");
+  static_assert(B(4, 11).*&B::m == 11, "");
+  static_assert(B(4, 11).m == 11, "");
+  static_assert(B(4, 11).*(int(A::*))&B::m == 11, "");
+  static_assert(B(4, 11).*&B::m == 11, "");
+  static_assert((&b)->*&A::n == 9, "");
+  static_assert((&b)->*&B::m == 13, "");
+  static_assert((&b)->*(int(A::*))&B::m == 13, "");
+  static_assert((B(4, 11).*&A::f)() == 7, "");
+  static_assert((B(4, 11).*&B::g)() == 16, "");
+
+  static_assert((B(4, 11).*(int(A::*)() const)&B::g)() == 16, "");
+
+  static_assert(((&b)->*&A::f)() == 12, "");
+  static_assert(((&b)->*&B::g)() == 23, "");
+  static_assert(((&b)->*(int(A::*)()const)&B::g)() == 23, "");
+
+
+  struct S {
+    constexpr S(int m, int n, int (S::*pf)() const, int S::*pn) :
+      m(m), n(n), pf(pf), pn(pn) {}
+    constexpr S() : m(), n(), pf(&S::f), pn(&S::n) {}
+
+    constexpr int f() const { return this->*pn; }
+    virtual int g() const;
+
+    int m, n;
+    int (S::*pf)() const;
+    int S::*pn;
+  };
+
+  constexpr int S::*pm = &S::m;
+  constexpr int S::*pn = &S::n;
+
+  constexpr int (S::*pf)() const = &S::f;
+  constexpr int (S::*pg)() const = &S::g;
+
+  constexpr S s(2, 5, &S::f, &S::m);
+
+  static_assert((s.*&S::f)() == 2, "");
+  static_assert((s.*s.pf)() == 2, "");
+
+  static_assert(pf == &S::f, "");
+
+  static_assert(pf == s.*&S::pf, "");
+
+  static_assert(pm == &S::m, "");
+  static_assert(pm != pn, "");
+  static_assert(s.pn != pn, "");
+  static_assert(s.pn == pm, "");
+  static_assert(pg != nullptr, "");
+  static_assert(pf != nullptr, "");
+  static_assert((int S::*)nullptr == nullptr, "");
+  static_assert(pg == pg, ""); // both-error {{constant expression}} \
+                               // both-note {{comparison of pointer to virtual member function 'g' has unspecified value}}
+  static_assert(pf != pg, ""); // both-error {{constant expression}} \
+                               // both-note {{comparison of pointer to virtual member function 'g' has unspecified value}}
+
+  template<int n> struct T : T<n-1> { const int X = n;};
+  template<> struct T<0> { int n; char k;};
+  template<> struct T<30> : T<29> { int m; };
+
+  T<17> t17;
+  T<30> t30;
+
+  constexpr int (T<15>::*deepm) = (int(T<10>::*))&T<30>::m;
+  constexpr int (T<10>::*deepn) = &T<0>::n;
+  constexpr char (T<10>::*deepk) = &T<0>::k;
+
+  static_assert(&(t17.*deepn) == &t17.n, "");
+  static_assert(&(t17.*deepk) == &t17.k, "");
+  static_assert(deepn == &T<2>::n, "");
+
+  constexpr int *pgood = &(t30.*deepm);
+  constexpr int *pbad = &(t17.*deepm); // both-error {{constant expression}}
+  static_assert(&(t30.*deepm) == &t30.m, "");
+
+  static_assert(deepm == &T<50>::m, "");
+  static_assert(deepm != deepn, "");
+
+  constexpr T<5> *p17_5 = &t17;
+  constexpr T<13> *p17_13 = (T<13>*)p17_5;
+  constexpr T<23> *p17_23 = (T<23>*)p17_13; // both-error {{constant expression}} \
+                                            // both-note {{cannot cast object of dynamic type 'T<17>' to type 'T<23>'}}
+  constexpr T<18> *p17_18 = (T<18>*)p17_13; // both-error {{constant expression}} \
+                                            // both-note {{cannot cast object of dynamic type 'T<17>' to type 'T<18>'}}
+  static_assert(&(p17_5->*(int(T<0>::*))deepn) == &t17.n, "");
+  static_assert(&(p17_5->*(int(T<0>::*))deepn), "");
+
+
+  static_assert(&(p17_13->*deepn) == &t17.n, "");
+  constexpr int *pbad2 = &(p17_13->*(int(T<9>::*))deepm); // both-error {{constant expression}}
+
+  constexpr T<5> *p30_5 = &t30;
+  constexpr T<23> *p30_23 = (T<23>*)p30_5;
+  constexpr T<13> *p30_13 = p30_23;
+  static_assert(&(p30_13->*deepn) == &t30.n, "");
+  static_assert(&(p30_23->*deepn) == &t30.n, "");
+  static_assert(&(p30_5->*(int(T<3>::*))deepn) == &t30.n, "");
+
+  static_assert(&(p30_5->*(int(T<2>::*))deepm) == &t30.m, "");
+  static_assert(&(((T<17>*)p30_13)->*deepm) == &t30.m, "");
+  static_assert(&(p30_23->*deepm) == &t30.m, "");
+
+
+  /// Added tests not from constant-expression-cxx11.cpp
+  static_assert(pm, "");
+  static_assert(!((int S::*)nullptr), "");
+  constexpr int S::*pk = nullptr;
+  static_assert(!pk, "");
+}
+
+namespace test3 {
+  struct nsCSSRect {
+  };
+  static int nsCSSRect::* sides;
+  nsCSSRect dimenX;
+  void ParseBoxCornerRadii(int y) {
+    switch (y) {
+    }
+    int& x = dimenX.*sides;
+  }
+}
+
+void foo() {
+  class X;
+  void (X::*d) ();
+  d = nullptr; /// This calls in the constant interpreter.
+}
+
+namespace {
+  struct A { int n; };
+  struct B { int n; };
+  struct C : A, B {};
+  struct D { double d; C c; };
+  const int &&u = static_cast<B&&>(0, ((D&&)D{}).*&D::c).n; // both-warning {{left operand of comma operator has no effect}}
+}
+
+/// From SemaTemplate/instantiate-member-pointers.cpp
+namespace {
+  struct Y {
+    int x;
+  };
+
+  template<typename T, typename Class, T Class::*Ptr>
+  struct X3 {
+    X3<T, Class, Ptr> &operator=(const T& value) {
+      return *this;
+    }
+  };
+
+  typedef int Y::*IntMember;
+  template<IntMember Member>
+  struct X4 {
+    X3<int, Y, Member> member;
+    int &getMember(Y& y) { return y.*Member; }
+  };
+
+  int &get_X4(X4<&Y::x> x4, Y& y) {
+    return x4.getMember(y);
+  }
+}
+
+/// From test/CXX/basic/basic.def.odr/p2.cpp
+namespace {
+  void use(int);
+  struct S { int x; int f() const; };
+  constexpr S *ps = nullptr;
+  S *const &psr = ps;
+
+  void test() {
+    use(ps->*&S::x);
+    use(psr->*&S::x);
+  }
+}
diff --git a/clang/test/AST/ast-dump-default-init-json.cpp b/clang/test/AST/ast-dump-default-init-json.cpp
index f4949a9c9eedf4f5ac024ac9660c11f57a39b89d..1058b4e3ea4d93d9cbd6003034568a0d038ceb5f 100644
--- a/clang/test/AST/ast-dump-default-init-json.cpp
+++ b/clang/test/AST/ast-dump-default-init-json.cpp
@@ -789,10 +789,10 @@ void test() {
 // CHECK-NEXT:                  "valueCategory": "lvalue",
 // CHECK-NEXT:                  "extendingDecl": {
 // CHECK-NEXT:                   "id": "0x{{.*}}",
-// CHECK-NEXT:                   "kind": "VarDecl",
-// CHECK-NEXT:                   "name": "b",
+// CHECK-NEXT:                   "kind": "FieldDecl",
+// CHECK-NEXT:                   "name": "a",
 // CHECK-NEXT:                   "type": {
-// CHECK-NEXT:                    "qualType": "B"
+// CHECK-NEXT:                    "qualType": "const A &"
 // CHECK-NEXT:                   }
 // CHECK-NEXT:                  },
 // CHECK-NEXT:                  "storageDuration": "automatic",
diff --git a/clang/test/AST/ast-dump-default-init.cpp b/clang/test/AST/ast-dump-default-init.cpp
index 26864fbf15424dc6af8815e3cdc02d836f111b51..15b29f04bf21bf6700096a2f1f03577e9d95f312 100644
--- a/clang/test/AST/ast-dump-default-init.cpp
+++ b/clang/test/AST/ast-dump-default-init.cpp
@@ -13,7 +13,7 @@ void test() {
 }
 // CHECK: -CXXDefaultInitExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue has rewritten init
 // CHECK-NEXT:  `-ExprWithCleanups 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue
-// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Var 0x{{[^ ]*}} 'b' 'B'
+// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Field 0x{{[^ ]*}} 'a' 'const A &'
 // CHECK-NEXT:      `-ImplicitCastExpr 0x{{[^ ]*}} <{{.*}}> 'const A' <NoOp>
 // CHECK-NEXT:        `-CXXFunctionalCastExpr 0x{{[^ ]*}} <{{.*}}> 'A' functional cast to A <NoOp>
 // CHECK-NEXT:          `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'A'
diff --git a/clang/test/Analysis/casts.c b/clang/test/Analysis/casts.c
index 30cd74be564fdae7a1d3468b937ded10fd848f73..7dad4edfa89b99ac142ee0cfc567174ce398aca7 100644
--- a/clang/test/Analysis/casts.c
+++ b/clang/test/Analysis/casts.c
@@ -138,7 +138,9 @@ void multiDimensionalArrayPointerCasts(void) {
   clang_analyzer_eval(y1 == y2); // expected-warning{{TRUE}}
 
   // FIXME: should be FALSE (i.e. equal pointers).
+  // FIXME: pointer subtraction warning might be incorrect
   clang_analyzer_eval(y1 - y2); // expected-warning{{UNKNOWN}}
+  // expected-warning@-1{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
   // FIXME: should be TRUE (i.e. same symbol).
   clang_analyzer_eval(*y1 == *y2); // expected-warning{{UNKNOWN}}
 
@@ -147,7 +149,9 @@ void multiDimensionalArrayPointerCasts(void) {
   clang_analyzer_eval(y1 == y3); // expected-warning{{TRUE}}
 
   // FIXME: should be FALSE (i.e. equal pointers).
+  // FIXME: pointer subtraction warning might be incorrect
   clang_analyzer_eval(y1 - y3); // expected-warning{{UNKNOWN}}
+  // expected-warning@-1{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
   // FIXME: should be TRUE (i.e. same symbol).
   clang_analyzer_eval(*y1 == *y3); // expected-warning{{UNKNOWN}}
 
diff --git a/clang/test/Analysis/cxx-uninitialized-object.cpp b/clang/test/Analysis/cxx-uninitialized-object.cpp
index aee0dae15fbfa4b2425da87f559ae69ca20a8b73..e3fa8ae8d7f2957f62c280c6b6b4c9ab1238e6ae 100644
--- a/clang/test/Analysis/cxx-uninitialized-object.cpp
+++ b/clang/test/Analysis/cxx-uninitialized-object.cpp
@@ -1114,27 +1114,27 @@ void fCXX11MemberInitTest1() {
   CXX11MemberInitTest1();
 }
 
-#ifdef PEDANTIC
 struct CXX11MemberInitTest2 {
   struct RecordType {
-    int a; // expected-note {{uninitialized field 'this->a'}}
-    int b; // expected-note {{uninitialized field 'this->b'}}
+    // TODO: we'd expect the note: {{uninitialized field 'this->rec.a'}}
+    int a; // no-note
+    // TODO: we'd expect the note: {{uninitialized field 'this->rec.b'}}
+    int b; // no-note
 
     RecordType(int) {}
   };
 
-  RecordType rec = RecordType(int()); // expected-warning {{2 uninitialized fields}}
+  RecordType rec = RecordType(int());
   int dontGetFilteredByNonPedanticMode = 0;
 
   CXX11MemberInitTest2() {}
 };
 
 void fCXX11MemberInitTest2() {
+  // TODO: we'd expect the warning: {{2 uninitializeds field}}
   CXX11MemberInitTest2(); // no-warning
 }
 
-#endif // PEDANTIC
-
 //===----------------------------------------------------------------------===//
 // "Esoteric" primitive type tests.
 //===----------------------------------------------------------------------===//
diff --git a/clang/test/Analysis/lifetime-extended-regions.cpp b/clang/test/Analysis/lifetime-extended-regions.cpp
index 524f4e0c400d17519b5715f3ee20e62cfad0d8da..4e98bd4b0403ebc89ea84fee70424cd18a5db16c 100644
--- a/clang/test/Analysis/lifetime-extended-regions.cpp
+++ b/clang/test/Analysis/lifetime-extended-regions.cpp
@@ -120,11 +120,11 @@ void aggregateWithReferences() {
   clang_analyzer_dump(viaReference);    // expected-warning-re {{&lifetime_extended_object{RefAggregate, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.rx); // expected-warning-re {{&lifetime_extended_object{int, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.ry); // expected-warning-re {{&lifetime_extended_object{Composite, viaReference, S{{[0-9]+}}} }}
-  
-  // The lifetime lifetime of object bound to reference members of aggregates,
-  // that are created from default member initializer was extended.
-  RefAggregate defaultInitExtended{i};
-  clang_analyzer_dump(defaultInitExtended.ry); // expected-warning-re {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }}
+
+  // clang does not currently implement extending lifetime of object bound to reference members of aggregates,
+  // that are created from default member initializer (see `warn_unsupported_lifetime_extension` from `-Wdangling`)
+  RefAggregate defaultInitExtended{i}; // clang-bug does not extend `Composite`
+  clang_analyzer_dump(defaultInitExtended.ry); // expected-warning {{Unknown }}
 }
 
 void lambda() {
diff --git a/clang/test/Analysis/pointer-sub.c b/clang/test/Analysis/pointer-sub.c
new file mode 100644
index 0000000000000000000000000000000000000000..9a446547e2868fc5d26b7471cb01e37c2e640699
--- /dev/null
+++ b/clang/test/Analysis/pointer-sub.c
@@ -0,0 +1,120 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.PointerSub -verify %s
+
+void f1(void) {
+  int x, y, z[10];
+  int d = &y - &x; // expected-warning{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
+  d = z - &y; // expected-warning{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
+  d = &x - &x; // no-warning (subtraction of any two identical pointers is allowed)
+  d = (long *)&x - (long *)&x;
+  d = (&x + 1) - &x; // no-warning ('&x' is like a single-element array)
+  d = &x - (&x + 1); // no-warning
+  d = (&x + 0) - &x; // no-warning
+  d = (&x - 1) - &x; // expected-warning{{Indexing the address of a variable with other than 1 at this place is undefined behavior}}
+  d = (&x + 2) - &x; // expected-warning{{Indexing the address of a variable with other than 1 at this place is undefined behavior}}
+
+  d = (z + 9) - z; // no-warning (pointers to same array)
+  d = (z + 10) - z; // no-warning (pointer to "one after the end")
+  d = (z + 11) - z; // expected-warning{{Using an array index greater than the array size at pointer subtraction is undefined behavior}}
+  d = (z - 1) - z; // expected-warning{{Using a negative array index at pointer subtraction is undefined behavior}}
+}
+
+void f2(void) {
+  int a[10], b[10], c;
+  int *p = &a[2];
+  int *q = &a[8];
+  int d = q - p; // no-warning (pointers into the same array)
+
+  q = &b[3];
+  d = q - p; // expected-warning{{Subtraction of two pointers that}}
+
+  q = a + 10;
+  d = q - p; // no warning (use of pointer to one after the end is allowed)
+  q = a + 11;
+  d = q - a; // expected-warning{{Using an array index greater than the array size at pointer subtraction is undefined behavior}}
+
+  d = &a[4] - a; // no-warning
+  d = &a[2] - p; // no-warning
+  d = &c - p; // expected-warning{{Subtraction of two pointers that}}
+
+  d = (int *)((char *)(&a[4]) + sizeof(int)) - &a[4]; // no-warning (pointers into the same array data)
+  d = (int *)((char *)(&a[4]) + 1) - &a[4]; // expected-warning{{Subtraction of two pointers that}}
+}
+
+void f3(void) {
+  int a[3][4];
+  int d;
+
+  d = &(a[2]) - &(a[1]);
+  d = a[2] - a[1]; // expected-warning{{Subtraction of two pointers that}}
+  d = a[1] - a[1];
+  d = &(a[1][2]) - &(a[1][0]);
+  d = &(a[1][2]) - &(a[0][0]); // expected-warning{{Subtraction of two pointers that}}
+
+  d = (int *)((char *)(&a[2][2]) + sizeof(int)) - &a[2][2]; // expected-warning{{Subtraction of two pointers that}}
+  d = (int *)((char *)(&a[2][2]) + 1) - &a[2][2]; // expected-warning{{Subtraction of two pointers that}}
+  d = (int (*)[4])((char *)&a[2] + sizeof(int (*)[4])) - &a[2]; // expected-warning{{Subtraction of two pointers that}}
+  d = (int (*)[4])((char *)&a[2] + 1) - &a[2]; // expected-warning{{Subtraction of two pointers that}}
+}
+
+void f4(void) {
+  int n = 4, m = 3;
+  int a[n][m];
+  int (*p)[m] = a; // p == &a[0]
+  p += 1; // p == &a[1]
+
+  // FIXME: This is a known problem with -Wpointer-arith (https://github.com/llvm/llvm-project/issues/28328)
+  int d = p - a; // d == 1 // expected-warning{{subtraction of pointers to type 'int[m]' of zero size has undefined behavior}}
+
+  // FIXME: This is a known problem with -Wpointer-arith (https://github.com/llvm/llvm-project/issues/28328)
+  d = &(a[2]) - &(a[1]); // expected-warning{{subtraction of pointers to type 'int[m]' of zero size has undefined behavior}}
+
+  d = a[2] - a[1]; // expected-warning{{Subtraction of two pointers that}}
+}
+
+typedef struct {
+  int a;
+  int b;
+  int c[10];
+  int d[10];
+} S;
+
+void f5(void) {
+  S s;
+  int y;
+  int d;
+
+  d = &s.b - &s.a; // expected-warning{{Subtraction of two pointers that}}
+  d = &s.c[0] - &s.a; // expected-warning{{Subtraction of two pointers that}}
+  d = &s.b - &y; // expected-warning{{Subtraction of two pointers that}}
+  d = &s.c[3] - &s.c[2];
+  d = &s.d[3] - &s.c[2]; // expected-warning{{Subtraction of two pointers that}}
+  d = s.d - s.c; // expected-warning{{Subtraction of two pointers that}}
+
+  S sa[10];
+  d = &sa[2] - &sa[1];
+  d = &sa[2].a - &sa[1].b; // expected-warning{{Subtraction of two pointers that}}
+}
+
+void f6(void) {
+  long long l;
+  char *a1 = (char *)&l;
+  int d = a1[3] - l;
+
+  long long la1[3];
+  long long la2[3];
+  char *pla1 = (char *)la1;
+  char *pla2 = (char *)la2;
+  d = pla1[1] - pla1[0];
+  d = (long long *)&pla1[1] - &l; // expected-warning{{Subtraction of two pointers that}}
+  d = &pla2[3] - &pla1[3]; // expected-warning{{Subtraction of two pointers that}}
+}
+
+void f7(int *p) {
+  int a[10];
+  int d = &a[10] - p; // no-warning ('p' is unknown, even if it cannot point into 'a')
+}
+
+void f8(int n) {
+  int a[10];
+  int d = a[n] - a[0];
+}
diff --git a/clang/test/Analysis/ptr-arith.c b/clang/test/Analysis/ptr-arith.c
index 40c8188704e811bad005ecf176fb94c30129b3a8..f99dfabb0736666399c900410ca27ddf82cb17ff 100644
--- a/clang/test/Analysis/ptr-arith.c
+++ b/clang/test/Analysis/ptr-arith.c
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,alpha.core.PointerSub,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple x86_64-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,alpha.core.PointerSub,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple i686-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple x86_64-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple i686-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
 
 void clang_analyzer_eval(int);
 void clang_analyzer_dump(int);
@@ -35,16 +35,6 @@ domain_port (const char *domain_b, const char *domain_e,
   return port;
 }
 
-void f3(void) {
-  int x, y;
-  int d = &y - &x; // expected-warning{{Subtraction of two pointers that do not point to the same memory chunk may cause incorrect result}}
-
-  int a[10];
-  int *p = &a[2];
-  int *q = &a[8];
-  d = q-p; // no-warning
-}
-
 void f4(void) {
   int *p;
   p = (int*) 0x10000; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp
index a908518f02ea2e49065b1ce15fe4c82aae91e640..c58cbfefc7997c6a2577c53697c883b29390d96d 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp
@@ -9,7 +9,7 @@ namespace test0 {
     typedef int A; // expected-note {{declared here}}
 
     int test() {
-      struct A a; // expected-error {{typedef 'A' cannot be referenced with a struct specifier}}
+      struct A a; // expected-error {{typedef 'A' cannot be referenced with the 'struct' specifier}}
       return a.foo;
     }
   }
@@ -18,7 +18,7 @@ namespace test0 {
     template <class> class A; // expected-note {{declared here}}
 
     int test() {
-      struct A a; // expected-error {{template 'A' cannot be referenced with a struct specifier}}
+      struct A a; // expected-error {{template 'A' cannot be referenced with the 'struct' specifier}}
       return a.foo;
     }
   }
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p6-cxx11.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p6-cxx11.cpp
index 97b2953b903123c2529f5b8882702db3a009cdd4..473290dd1c191a120080f5b05d5186ea550bdcd8 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p6-cxx11.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p6-cxx11.cpp
@@ -1,7 +1,10 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
 
 namespace A {
   namespace B { }
 }
 
+// CHECK: fix-it:"{{.*}}":{[[@LINE+1]]:7-[[@LINE+1]]:7}:"namespace "
 using A::B; // expected-error{{using declaration cannot refer to a namespace}}
+            // expected-note@-1 {{did you mean 'using namespace'?}}
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
index 4416c825226494b413c6c5887af470ac9a322d92..51990ee4341d29b5d1d0502d7d609275d3ee2bde 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
@@ -212,7 +212,7 @@ constexpr int ClassDecl3() {
   return 0;
 }
 
-constexpr int NoReturn() {} // expected-error {{no return statement in constexpr function}}
+constexpr int NoReturn() {} // beforecxx23-error {{no return statement in constexpr function}}
 constexpr int MultiReturn() {
   return 0; // beforecxx14-note {{return statement}}
   return 0; // beforecxx14-warning {{multiple return statements in constexpr function}}
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp
index f3e79c0aae44a8a85551c980d3515043254fb085..98c9b915c6ce9f0f5f577ebf4a6273ed4a09a4d0 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp
@@ -2,18 +2,18 @@
 
 struct A { typedef int type; };
 template<typename T> using X = A; // expected-note {{declared here}}
-struct X<int>* p2; // expected-error {{type alias template 'X' cannot be referenced with a struct specifier}}
+struct X<int>* p2; // expected-error {{alias template 'X' cannot be referenced with the 'struct' specifier}}
 
 
 template<typename T> using Id = T; // expected-note {{declared here}}
 template<template<typename> class F>
 struct Y {
-  struct F<int> i; // expected-error {{type alias template 'Id' cannot be referenced with a struct specifier}}
+  struct F<int> i; // expected-error {{alias template 'Id' cannot be referenced with the 'struct' specifier}}
   typename F<A>::type j; // ok
 
   // FIXME: don't produce the diagnostic both for the definition and the instantiation.
   template<typename T> using U = F<char>; // expected-note 2{{declared here}}
-  struct Y<F>::template U<char> k; // expected-error 2{{type alias template 'U' cannot be referenced with a struct specifier}}
+  struct Y<F>::template U<char> k; // expected-error 2{{alias template 'U' cannot be referenced with the 'struct' specifier}}
   typename Y<F>::template U<char> l; // ok
 };
 template struct Y<Id>; // expected-note {{requested here}}
diff --git a/clang/test/CXX/drs/cwg16xx.cpp b/clang/test/CXX/drs/cwg16xx.cpp
index 82ef871939d2c468efc3837384c8f53db695562d..cf6b45ceabf2cc8f85bea33900fe3d3a1a4375f8 100644
--- a/clang/test/CXX/drs/cwg16xx.cpp
+++ b/clang/test/CXX/drs/cwg16xx.cpp
@@ -483,6 +483,8 @@ namespace cwg1696 { // cwg1696: 7
     const A &a = A(); // #cwg1696-D1-a
   };
   D1 d1 = {}; // #cwg1696-d1
+  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}}
+  //   since-cxx14-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}}
 
   struct D2 {
     const A &a = A(); // #cwg1696-D2-a
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 054ce5a4f4b704fc2fb48e7c2dce9014c88e2a50..323e56f9c527876b1eac21ac68818c0240696fa9 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -206,28 +206,19 @@ namespace cwg1814 { // cwg1814: yes
 #endif
 }
 
-namespace cwg1815 { // cwg1815: 19
+namespace cwg1815 { // cwg1815: no
 #if __cplusplus >= 201402L
-  struct A { int &&r = 0; };
+  // FIXME: needs codegen test
+  struct A { int &&r = 0; }; // #cwg1815-A
   A a = {};
+  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} FIXME
+  //   since-cxx14-note@#cwg1815-A {{initializing field 'r' with default member initializer}}
 
   struct B { int &&r = 0; }; // #cwg1815-B
   // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
   //   since-cxx14-note@#cwg1815-B {{initializing field 'r' with default member initializer}}
   //   since-cxx14-note@#cwg1815-b {{in implicit default constructor for 'cwg1815::B' first required here}}
   B b; // #cwg1815-b
-
-#if __cplusplus >= 201703L
-  struct C { const int &r = 0; };
-  constexpr C c = {}; // OK, since cwg1815
-  static_assert(c.r == 0);
-
-  constexpr int f() {
-    A a = {}; // OK, since cwg1815
-    return a.r;
-  }
-  static_assert(f() == 0);
-#endif
 #endif
 }
 
diff --git a/clang/test/CXX/drs/cwg2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp
index 2b3131be33057a2ede491a115ec776189a86487d..99916dea9a9120be918f903d3078f72e4e037775 100644
--- a/clang/test/CXX/drs/cwg2xx.cpp
+++ b/clang/test/CXX/drs/cwg2xx.cpp
@@ -759,7 +759,7 @@ namespace cwg254 { // cwg254: 2.9
     typedef typename T::type type; // ok even if this is a typedef-name, because
                                    // it's not an elaborated-type-specifier
     typedef struct T::type foo;
-    // expected-error@-1 {{typedef 'type' cannot be referenced with a struct specifier}}
+    // expected-error@-1 {{typedef 'type' cannot be referenced with the 'struct' specifier}}
     //   expected-note@#cwg254-instantiation {{in instantiation of template class 'cwg254::A<cwg254::C>' requested here}}
     //   expected-note@#cwg254-C {{declared here}}
   };
@@ -1264,10 +1264,10 @@ namespace cwg298 { // cwg298: 3.1
 
   struct A a;
   struct B b;
-  // expected-error@-1 {{typedef 'B' cannot be referenced with a struct specifier}}
+  // expected-error@-1 {{typedef 'B' cannot be referenced with the 'struct' specifier}}
   //   expected-note@#cwg298-B {{declared here}}
   struct C c;
-  // expected-error@-1 {{typedef 'C' cannot be referenced with a struct specifier}}
+  // expected-error@-1 {{typedef 'C' cannot be referenced with the 'struct' specifier}}
   //   expected-note@#cwg298-C {{declared here}}
 
   B::B() {}
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index 07162cc28f6b6006606ca7cef1e725516f9bc7a5..98ff7553d989ba186813700c60d423b54288e59f 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -170,7 +170,7 @@ namespace cwg407 { // cwg407: 3.8
     {
       typedef struct S S; // #cwg407-typedef-S
       struct S *p;
-      // expected-error@-1 {{typedef 'S' cannot be referenced with a struct specifier}}
+      // expected-error@-1 {{typedef 'S' cannot be referenced with the 'struct' specifier}}
       //   expected-note@#cwg407-typedef-S {{declared here}}
     }
   }
@@ -941,8 +941,10 @@ namespace cwg460 { // cwg460: yes
     // expected-error@-1 {{using declaration requires a qualified name}}
     using cwg460::X;
     // expected-error@-1 {{using declaration cannot refer to a namespace}}
+    // expected-note@-2 {{did you mean 'using namespace'?}}
     using X::Q;
     // expected-error@-1 {{using declaration cannot refer to a namespace}}
+    // expected-note@-2 {{did you mean 'using namespace'?}}
   }
 }
 
diff --git a/clang/test/CXX/special/class.temporary/p6.cpp b/clang/test/CXX/special/class.temporary/p6.cpp
index a6d2adfd1fd2c5bac6d063cdd839011bb8828b13..5554363cc69abb5190169e6c181d3c84756cf83d 100644
--- a/clang/test/CXX/special/class.temporary/p6.cpp
+++ b/clang/test/CXX/special/class.temporary/p6.cpp
@@ -269,40 +269,6 @@ void init_capture_init_list() {
   // CHECK: }
 }
 
-void check_dr1815() { // dr1815: yes
-#if __cplusplus >= 201402L
-
-  struct A {
-    int &&r = 0;
-    ~A() {}
-  };
-
-  struct B {
-    A &&a = A{};
-    ~B() {}
-  };
-  B a = {};
-  
-  // CHECK: call {{.*}}block_scope_begin_function
-  extern void block_scope_begin_function();
-  extern void block_scope_end_function();
-  block_scope_begin_function();
-  {
-    // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
-    // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
-    B b = {};
-  }
-  // CHECK: call {{.*}}block_scope_end_function
-  block_scope_end_function();
-
-  // CHECK: call {{.*}}some_other_function
-  extern void some_other_function();
-  some_other_function();
-  // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
-  // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
-#endif
-}
-
 namespace P2718R0 {
 namespace basic {
 template <typename E> using T2 = std::list<E>;
diff --git a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp
index 1cf9e1c9f9c0fa39caf6c1d326e68519cee41a62..b8092afaffef3f7e812ae87af5419b9d55c8dad0 100644
--- a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp
@@ -174,7 +174,7 @@ namespace test7 {
 
   // This shouldn't crash.
   template <class T> class D {
-    friend class A; // expected-error {{template 'A' cannot be referenced with a class specifier}}
+    friend class A; // expected-error {{template 'A' cannot be referenced with the 'class' specifier}}
   };
   template class D<int>;
 }
diff --git a/clang/test/CXX/temp/temp.spec/no-body.cpp b/clang/test/CXX/temp/temp.spec/no-body.cpp
index 6d1b82fe1898a48fe091915d8e8c298365246baa..5b0c4be300338007830d02de0a143d2527f7194d 100644
--- a/clang/test/CXX/temp/temp.spec/no-body.cpp
+++ b/clang/test/CXX/temp/temp.spec/no-body.cpp
@@ -43,7 +43,7 @@ namespace good { // Only good in C++98/03
 
 namespace unsupported {
 #ifndef FIXING
- template struct y;     // expected-error {{template 'y' cannot be referenced with a struct specifier}}
+ template struct y;     // expected-error {{template 'y' cannot be referenced with the 'struct' specifier}}
 #endif
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c
index 13748be1acc1ac9b2f04540e1e41a47cafc5addc..b87b225b632a5edc23eecbca3f5a59631606f0e1 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c
@@ -16,399 +16,399 @@
 
 #include <riscv_vector.h>
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8(
+// CHECK-LABEL: @test_vcpop_v_u8mf8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.nxv1i8.i64(<vscale x 1 x i8> poison, <vscale x 1 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8(vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8(vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8(vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4(
+// CHECK-LABEL: @test_vcpop_v_u8mf4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.nxv2i8.i64(<vscale x 2 x i8> poison, <vscale x 2 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4(vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4(vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4(vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2(
+// CHECK-LABEL: @test_vcpop_v_u8mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.nxv4i8.i64(<vscale x 4 x i8> poison, <vscale x 4 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2(vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2(vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2(vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1(
+// CHECK-LABEL: @test_vcpop_v_u8m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.nxv8i8.i64(<vscale x 8 x i8> poison, <vscale x 8 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1(vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1(vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1(vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2(
+// CHECK-LABEL: @test_vcpop_v_u8m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.nxv16i8.i64(<vscale x 16 x i8> poison, <vscale x 16 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2(vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2(vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2(vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4(
+// CHECK-LABEL: @test_vcpop_v_u8m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.nxv32i8.i64(<vscale x 32 x i8> poison, <vscale x 32 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4(vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4(vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4(vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8(
+// CHECK-LABEL: @test_vcpop_v_u8m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.nxv64i8.i64(<vscale x 64 x i8> poison, <vscale x 64 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8(vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8(vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8(vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4(
+// CHECK-LABEL: @test_vcpop_v_u16mf4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.nxv1i16.i64(<vscale x 1 x i16> poison, <vscale x 1 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4(vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4(vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4(vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2(
+// CHECK-LABEL: @test_vcpop_v_u16mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.nxv2i16.i64(<vscale x 2 x i16> poison, <vscale x 2 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2(vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2(vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2(vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1(
+// CHECK-LABEL: @test_vcpop_v_u16m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.nxv4i16.i64(<vscale x 4 x i16> poison, <vscale x 4 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1(vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1(vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1(vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2(
+// CHECK-LABEL: @test_vcpop_v_u16m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.nxv8i16.i64(<vscale x 8 x i16> poison, <vscale x 8 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2(vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2(vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2(vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4(
+// CHECK-LABEL: @test_vcpop_v_u16m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.nxv16i16.i64(<vscale x 16 x i16> poison, <vscale x 16 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4(vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4(vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4(vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8(
+// CHECK-LABEL: @test_vcpop_v_u16m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.nxv32i16.i64(<vscale x 32 x i16> poison, <vscale x 32 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8(vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8(vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8(vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2(
+// CHECK-LABEL: @test_vcpop_v_u32mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.nxv1i32.i64(<vscale x 1 x i32> poison, <vscale x 1 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2(vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2(vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2(vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1(
+// CHECK-LABEL: @test_vcpop_v_u32m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1(vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1(vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1(vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2(
+// CHECK-LABEL: @test_vcpop_v_u32m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.nxv4i32.i64(<vscale x 4 x i32> poison, <vscale x 4 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2(vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2(vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2(vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4(
+// CHECK-LABEL: @test_vcpop_v_u32m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4(vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4(vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4(vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8(
+// CHECK-LABEL: @test_vcpop_v_u32m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8(vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8(vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8(vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1(
+// CHECK-LABEL: @test_vcpop_v_u64m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.nxv1i64.i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1(vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1(vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1(vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2(
+// CHECK-LABEL: @test_vcpop_v_u64m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.nxv2i64.i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2(vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2(vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2(vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4(
+// CHECK-LABEL: @test_vcpop_v_u64m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.nxv4i64.i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4(vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4(vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4(vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8(
+// CHECK-LABEL: @test_vcpop_v_u64m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.nxv8i64.i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8(vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8(vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8(vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> poison, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8_m(mask, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> poison, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4_m(mask, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> poison, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2_m(mask, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_m(
+// CHECK-LABEL: @test_vcpop_v_u8m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> poison, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1_m(mask, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_m(
+// CHECK-LABEL: @test_vcpop_v_u8m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> poison, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2_m(mask, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_m(
+// CHECK-LABEL: @test_vcpop_v_u8m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> poison, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4_m(mask, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_m(
+// CHECK-LABEL: @test_vcpop_v_u8m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> poison, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8_m(mask, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_m(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> poison, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4_m(mask, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> poison, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2_m(mask, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_m(
+// CHECK-LABEL: @test_vcpop_v_u16m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> poison, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1_m(mask, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_m(
+// CHECK-LABEL: @test_vcpop_v_u16m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> poison, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2_m(mask, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_m(
+// CHECK-LABEL: @test_vcpop_v_u16m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> poison, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4_m(mask, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_m(
+// CHECK-LABEL: @test_vcpop_v_u16m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> poison, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8_m(mask, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> poison, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2_m(mask, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_m(
+// CHECK-LABEL: @test_vcpop_v_u32m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1_m(mask, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_m(
+// CHECK-LABEL: @test_vcpop_v_u32m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> poison, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2_m(mask, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_m(
+// CHECK-LABEL: @test_vcpop_v_u32m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4_m(mask, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_m(
+// CHECK-LABEL: @test_vcpop_v_u32m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8_m(mask, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_m(
+// CHECK-LABEL: @test_vcpop_v_u64m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1_m(mask, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_m(
+// CHECK-LABEL: @test_vcpop_v_u64m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2_m(mask, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_m(
+// CHECK-LABEL: @test_vcpop_v_u64m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4_m(mask, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4_m(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_m(
+// CHECK-LABEL: @test_vcpop_v_u64m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8_m(mask, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8_m(mask, vs2, vl);
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c
index adb0ac9ee5d79a3e4050a2fb5998d58719551be1..5625b19f57f3ad6230a1a81bc6fc73c9e51e066a 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c
@@ -16,399 +16,399 @@
 
 #include <riscv_vector.h>
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8(
+// CHECK-LABEL: @test_vcpop_v_u8mf8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.nxv1i8.i64(<vscale x 1 x i8> poison, <vscale x 1 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8(vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8(vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4(
+// CHECK-LABEL: @test_vcpop_v_u8mf4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.nxv2i8.i64(<vscale x 2 x i8> poison, <vscale x 2 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4(vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4(vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2(
+// CHECK-LABEL: @test_vcpop_v_u8mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.nxv4i8.i64(<vscale x 4 x i8> poison, <vscale x 4 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2(vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2(vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1(
+// CHECK-LABEL: @test_vcpop_v_u8m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.nxv8i8.i64(<vscale x 8 x i8> poison, <vscale x 8 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1(vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1(vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2(
+// CHECK-LABEL: @test_vcpop_v_u8m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.nxv16i8.i64(<vscale x 16 x i8> poison, <vscale x 16 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2(vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2(vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4(
+// CHECK-LABEL: @test_vcpop_v_u8m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.nxv32i8.i64(<vscale x 32 x i8> poison, <vscale x 32 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4(vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4(vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8(
+// CHECK-LABEL: @test_vcpop_v_u8m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.nxv64i8.i64(<vscale x 64 x i8> poison, <vscale x 64 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8(vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8(vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4(
+// CHECK-LABEL: @test_vcpop_v_u16mf4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.nxv1i16.i64(<vscale x 1 x i16> poison, <vscale x 1 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4(vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4(vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2(
+// CHECK-LABEL: @test_vcpop_v_u16mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.nxv2i16.i64(<vscale x 2 x i16> poison, <vscale x 2 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2(vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2(vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1(
+// CHECK-LABEL: @test_vcpop_v_u16m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.nxv4i16.i64(<vscale x 4 x i16> poison, <vscale x 4 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1(vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1(vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2(
+// CHECK-LABEL: @test_vcpop_v_u16m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.nxv8i16.i64(<vscale x 8 x i16> poison, <vscale x 8 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2(vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2(vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4(
+// CHECK-LABEL: @test_vcpop_v_u16m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.nxv16i16.i64(<vscale x 16 x i16> poison, <vscale x 16 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4(vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4(vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8(
+// CHECK-LABEL: @test_vcpop_v_u16m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.nxv32i16.i64(<vscale x 32 x i16> poison, <vscale x 32 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8(vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8(vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2(
+// CHECK-LABEL: @test_vcpop_v_u32mf2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.nxv1i32.i64(<vscale x 1 x i32> poison, <vscale x 1 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2(vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2(vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1(
+// CHECK-LABEL: @test_vcpop_v_u32m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1(vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1(vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2(
+// CHECK-LABEL: @test_vcpop_v_u32m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.nxv4i32.i64(<vscale x 4 x i32> poison, <vscale x 4 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2(vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2(vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4(
+// CHECK-LABEL: @test_vcpop_v_u32m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4(vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4(vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8(
+// CHECK-LABEL: @test_vcpop_v_u32m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8(vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8(vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1(
+// CHECK-LABEL: @test_vcpop_v_u64m1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.nxv1i64.i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1(vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1(vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2(
+// CHECK-LABEL: @test_vcpop_v_u64m2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.nxv2i64.i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2(vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2(vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4(
+// CHECK-LABEL: @test_vcpop_v_u64m4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.nxv4i64.i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4(vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4(vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8(
+// CHECK-LABEL: @test_vcpop_v_u64m8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.nxv8i64.i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8(vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8(vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop(vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> poison, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> poison, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> poison, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_m(
+// CHECK-LABEL: @test_vcpop_v_u8m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> poison, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_m(
+// CHECK-LABEL: @test_vcpop_v_u8m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> poison, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_m(
+// CHECK-LABEL: @test_vcpop_v_u8m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> poison, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_m(
+// CHECK-LABEL: @test_vcpop_v_u8m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> poison, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_m(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> poison, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> poison, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_m(
+// CHECK-LABEL: @test_vcpop_v_u16m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> poison, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_m(
+// CHECK-LABEL: @test_vcpop_v_u16m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> poison, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_m(
+// CHECK-LABEL: @test_vcpop_v_u16m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> poison, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_m(
+// CHECK-LABEL: @test_vcpop_v_u16m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> poison, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_m(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> poison, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_m(
+// CHECK-LABEL: @test_vcpop_v_u32m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_m(
+// CHECK-LABEL: @test_vcpop_v_u32m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> poison, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_m(
+// CHECK-LABEL: @test_vcpop_v_u32m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_m(
+// CHECK-LABEL: @test_vcpop_v_u32m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_m(
+// CHECK-LABEL: @test_vcpop_v_u64m1_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_m(
+// CHECK-LABEL: @test_vcpop_v_u64m2_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_m(
+// CHECK-LABEL: @test_vcpop_v_u64m4_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_m(
+// CHECK-LABEL: @test_vcpop_v_u64m8_m(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 3)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv(mask, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop(mask, vs2, vl);
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c
index 8a1f2e1beec1107121657d48f08161c7f1ce969e..3a110339b5f18f05bb918358a743695b28d1169b 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c
@@ -16,795 +16,795 @@
 
 #include <riscv_vector.h>
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8_tu(maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4_tu(maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2_tu(maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1_tu(maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2_tu(maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4_tu(maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8_tu(maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4_tu(maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2_tu(maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1_tu(maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2_tu(maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4_tu(maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8_tu(maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2_tu(maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1_tu(maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2_tu(maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4_tu(maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8_tu(maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1_tu(maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2_tu(maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4_tu(maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8_tu(maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8_tum(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4_tum(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2_tum(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1_tum(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2_tum(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4_tum(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8_tum(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tum(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4_tum(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2_tum(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1_tum(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2_tum(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4_tum(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8_tum(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2_tum(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1_tum(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2_tum(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4_tum(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8_tum(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1_tum(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2_tum(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4_tum(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8_tum(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8_tumu(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4_tumu(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2_tumu(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1_tumu(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2_tumu(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4_tumu(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8_tumu(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4_tumu(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2_tumu(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1_tumu(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2_tumu(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4_tumu(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8_tumu(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2_tumu(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1_tumu(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2_tumu(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4_tumu(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8_tumu(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1_tumu(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2_tumu(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4_tumu(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8_tumu(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf8_mu(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf8_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf4_mu(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8mf2_mu(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8mf2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m1_mu(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m1_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m2_mu(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m4_mu(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u8m8_mu(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u8m8_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_mu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf4_mu(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16mf2_mu(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16mf2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m1_mu(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m1_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m2_mu(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m4_mu(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u16m8_mu(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u16m8_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32mf2_mu(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32mf2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m1_mu(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m1_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m2_mu(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m4_mu(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u32m8_mu(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u32m8_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m1_mu(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m1_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m2_mu(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m2_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m4_mu(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m4_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_v_u64m8_mu(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_v_u64m8_mu(mask, maskedoff, vs2, vl);
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c
index 02a499d4b67da54903b880f6a26efc0dfc7f17a5..953ccac133c382c5ae52147376d1923587bf9765 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c
@@ -16,795 +16,795 @@
 
 #include <riscv_vector.h>
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 [[VL:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tu(maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tu(maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tum(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tum(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tum(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 2)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tum(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_tumu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_tumu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf8_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vcpopv.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
-vuint8mf8_t test_vcpopv_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8mf8_t test_vcpop_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf4_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vcpopv.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
-vuint8mf4_t test_vcpopv_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8mf4_t test_vcpop_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u8mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vcpopv.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
-vuint8mf2_t test_vcpopv_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8mf2_t test_vcpop_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vcpopv.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
-vuint8m1_t test_vcpopv_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8m1_t test_vcpop_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vcpopv.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-vuint8m2_t test_vcpopv_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8m2_t test_vcpop_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vcpopv.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
-vuint8m4_t test_vcpopv_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8m4_t test_vcpop_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u8m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u8m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vcpopv.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
-vuint8m8_t test_vcpopv_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint8m8_t test_vcpop_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf4_mu(
+// CHECK-LABEL: @test_vcpop_v_u16mf4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vcpopv.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
-vuint16mf4_t test_vcpopv_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16mf4_t test_vcpop_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u16mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vcpopv.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
-vuint16mf2_t test_vcpopv_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16mf2_t test_vcpop_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vcpopv.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
-vuint16m1_t test_vcpopv_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16m1_t test_vcpop_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vcpopv.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
-vuint16m2_t test_vcpopv_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16m2_t test_vcpop_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vcpopv.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
-vuint16m4_t test_vcpopv_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16m4_t test_vcpop_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u16m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u16m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vcpopv.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
-vuint16m8_t test_vcpopv_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint16m8_t test_vcpop_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32mf2_mu(
+// CHECK-LABEL: @test_vcpop_v_u32mf2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vcpopv.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
-vuint32mf2_t test_vcpopv_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint32mf2_t test_vcpop_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vcpopv.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
-vuint32m1_t test_vcpopv_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint32m1_t test_vcpop_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vcpopv.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
-vuint32m2_t test_vcpopv_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint32m2_t test_vcpop_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vcpopv.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
-vuint32m4_t test_vcpopv_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint32m4_t test_vcpop_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u32m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u32m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vcpopv.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
-vuint32m8_t test_vcpopv_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint32m8_t test_vcpop_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m1_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m1_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vcpopv.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
-vuint64m1_t test_vcpopv_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint64m1_t test_vcpop_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m2_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m2_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vcpopv.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
-vuint64m2_t test_vcpopv_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint64m2_t test_vcpop_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m4_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m4_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vcpopv.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
-vuint64m4_t test_vcpopv_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint64m4_t test_vcpop_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
-// CHECK-LABEL: @test_vcpopv_v_u64m8_mu(
+// CHECK-LABEL: @test_vcpop_v_u64m8_mu(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vcpopv.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
-vuint64m8_t test_vcpopv_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
-  return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl);
+vuint64m8_t test_vcpop_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) {
+  return __riscv_vcpop_mu(mask, maskedoff, vs2, vl);
 }
 
diff --git a/clang/test/CodeGen/aarch64-cpu-supports-target.c b/clang/test/CodeGen/aarch64-cpu-supports-target.c
index e023944b24e53a1e80a0beb188fd22496e565fc7..28187bcf7453315da2273649e92c4269725a393f 100644
--- a/clang/test/CodeGen/aarch64-cpu-supports-target.c
+++ b/clang/test/CodeGen/aarch64-cpu-supports-target.c
@@ -48,5 +48,5 @@ int test_versions() {
     return code();
 }
 // CHECK: attributes #0 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #1 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon" }
-// CHECK: attributes #2 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #1 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
+// CHECK: attributes #2 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" }
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/aarch64-sme-attrs.cpp b/clang/test/CodeGen/aarch64-sme-intrinsics/aarch64-sme-attrs.cpp
index af8933d93d6cbb36d52c5be715abcd0b69644ed5..9885ac45e6a0e07cdd285fde64a0a6ed027da533 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/aarch64-sme-attrs.cpp
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/aarch64-sme-attrs.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme \
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +bf16 \
 // RUN:   -disable-O0-optnone -Werror -emit-llvm -o - %s \
 // RUN: | opt -S -passes=mem2reg \
 // RUN: | opt -S -passes=inline \
diff --git a/clang/test/CodeGen/aarch64-targetattr.c b/clang/test/CodeGen/aarch64-targetattr.c
index 3e7a209245607113d15ec1e4d4d82608eaf914cf..644e6a692c3be953fe107ee6292a21f7cc4fcb62 100644
--- a/clang/test/CodeGen/aarch64-targetattr.c
+++ b/clang/test/CodeGen/aarch64-targetattr.c
@@ -58,58 +58,50 @@ void v1msve() {}
 // CHECK-LABEL: @plussve() #12
 __attribute__((target("+sve")))
 void plussve() {}
-// CHECK-LABEL: @plussveplussve2() #13
+// CHECK-LABEL: @plussveplussve2() #12
 __attribute__((target("+sve+nosve2")))
 void plussveplussve2() {}
-// CHECK-LABEL: @plussveminusnosve2() #13
+// CHECK-LABEL: @plussveminusnosve2() #12
 __attribute__((target("sve,no-sve2")))
 void plussveminusnosve2() {}
-// CHECK-LABEL: @plusfp16() #14
+// CHECK-LABEL: @plusfp16() #13
 __attribute__((target("+fp16")))
 void plusfp16() {}
 
-// CHECK-LABEL: @all() #15
+// CHECK-LABEL: @all() #14
 __attribute__((target("cpu=neoverse-n1,tune=cortex-a710,arch=armv8.6-a+sve2")))
 void all() {}
-// CHECK-LABEL: @allplusbranchprotection() #16
+// CHECK-LABEL: @allplusbranchprotection() #15
 __attribute__((target("cpu=neoverse-n1,tune=cortex-a710,arch=armv8.6-a+sve2,branch-protection=standard")))
 void allplusbranchprotection() {}
 
-// These tests check that the user facing and internal llvm name are both accepted.
-// CHECK-LABEL: @plusnoneon() #17
-__attribute__((target("+noneon")))
-void plusnoneon() {}
-// CHECK-LABEL: @plusnosimd() #17
+// CHECK-LABEL: @plusnosimd() #16
 __attribute__((target("+nosimd")))
 void plusnosimd() {}
-// CHECK-LABEL: @noneon() #17
-__attribute__((target("no-neon")))
-void noneon() {}
-// CHECK-LABEL: @nosimd() #17
+// CHECK-LABEL: @nosimd() #16
 __attribute__((target("no-simd")))
 void nosimd() {}
 
 // This isn't part of the standard interface, but test that -arch features should not apply anything else.
-// CHECK-LABEL: @minusarch() #18
+// CHECK-LABEL: @minusarch() #17
 __attribute__((target("no-v9.3a")))
 void minusarch() {}
 
 // CHECK: attributes #0 = { {{.*}} "target-features"="+crc,+fp-armv8,+lse,+neon,+ras,+rdm,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #1 = { {{.*}} "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #2 = { {{.*}} "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8a" }
-// CHECK: attributes #3 = { {{.*}} "target-features"="+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
-// CHECK: attributes #4 = { {{.*}} "target-cpu"="cortex-a710" "target-features"="+bf16,+complxnum,+crc,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+ras,+rcpc,+rdm,+sb,+sve,+sve2,+sve2-bitperm" }
+// CHECK: attributes #3 = { {{.*}} "target-features"="+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
+// CHECK: attributes #4 = { {{.*}} "target-cpu"="cortex-a710" "target-features"="+bf16,+complxnum,+crc,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+ras,+rcpc,+rdm,+sb,+sve,+sve2,+sve2-bitperm,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a" }
 // CHECK: attributes #5 = { {{.*}} "tune-cpu"="cortex-a710" }
 // CHECK: attributes #6 = { {{.*}} "target-cpu"="generic" }
 // CHECK: attributes #7 = { {{.*}} "tune-cpu"="generic" }
-// CHECK: attributes #8 = { {{.*}} "target-cpu"="neoverse-n1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #9 = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+sve" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #10 = { {{.*}} "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2" }
-// CHECK: attributes #11 = { {{.*}} "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,-sve" }
-// CHECK: attributes #12 = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
-// CHECK: attributes #13 = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+sve,-sve2" }
-// CHECK: attributes #14 = { {{.*}} "target-features"="+fullfp16" }
-// CHECK: attributes #15 = { {{.*}} "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #16 = { {{.*}} "branch-target-enforcement"="true" "guarded-control-stack"="true" {{.*}} "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #17 = { {{.*}} "target-features"="-neon" }
-// CHECK: attributes #18 = { {{.*}} "target-features"="-v9.3a" }
+// CHECK: attributes #8 = { {{.*}} "target-cpu"="neoverse-n1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+v8.1a,+v8.2a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #9 = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+sve" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #10 = { {{.*}} "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+// CHECK: attributes #11 = { {{.*}} "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,-sve" }
+// CHECK: attributes #12 = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+sve" }
+// CHECK: attributes #13 = { {{.*}} "target-features"="+fp-armv8,+fullfp16" }
+// CHECK: attributes #14 = { {{.*}} "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #15 = { {{.*}} "branch-target-enforcement"="true" "guarded-control-stack"="true" {{.*}} "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK-NOT: attributes #16 = {{.*}} "target-features"
+// CHECK: attributes #17 = { {{.*}} "target-features"="-v9.3a" }
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index 3597711333d341887b85b9fbe6fdd64bd569681e..75f8734e5aaf373ffaa6941e58bd99e1b07a6680 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -1129,42 +1129,42 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NOFMV-NEXT:    ret i32 0
 //
 //.
-// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp-armv8,+fp16fml,+fullfp16,+neon,+rand,-v9.5a" }
 // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+sha2,-v9.5a" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+ls64,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fp16fml,+fullfp16,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,-v9.5a" }
 // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,-v9.5a" }
 // CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
 // CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme,-v9.5a" }
+// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
 // CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+jsconv,+neon,-v9.5a" }
 // CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+sve,-v9.5a" }
+// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-v9.5a" }
+// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-v9.5a" }
+// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-v9.5a" }
 // CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sm4,-v9.5a" }
+// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon,+rdm,-v9.5a" }
 //.
 // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
 // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
diff --git a/clang/test/CodeGen/debug-info-packed-struct.c b/clang/test/CodeGen/debug-info-packed-struct.c
index 6441a740e3799f32b66bbb6804201894d25a4ab7..676cdb38b396f29cc00b0f9724c17bfb00b50932 100644
--- a/clang/test/CodeGen/debug-info-packed-struct.c
+++ b/clang/test/CodeGen/debug-info-packed-struct.c
@@ -59,7 +59,7 @@ struct layout2 {
 #pragma pack()
 // CHECK: l2_ofs0
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "l2_ofs1",
-// CHECK-SAME:     {{.*}}size: 64, offset: 8)
+// CHECK-SAME:     {{.*}}size: 64, align: 8, offset: 8)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "l2_ofs9",
 // CHECK-SAME:     {{.*}}size: 1, offset: 72, flags: DIFlagBitField, extraData: i64 72)
 
@@ -81,7 +81,7 @@ struct layout3 {
 #pragma pack()
 // CHECK: l3_ofs0
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "l3_ofs4",
-// CHECK-SAME:     {{.*}}size: 64, offset: 32)
+// CHECK-SAME:     {{.*}}size: 64, align: 32, offset: 32)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "l3_ofs12",
 // CHECK-SAME:     {{.*}}size: 1, offset: 96, flags: DIFlagBitField, extraData: i64 96)
 
diff --git a/clang/test/CodeGen/instrument-objc-method.m b/clang/test/CodeGen/instrument-objc-method.m
index cfc0a0a98bec6b99e356f406fe815852e81841ef..2c9d1fc88554bdc3ababbbb0dbcac6bd802a69e6 100644
--- a/clang/test/CodeGen/instrument-objc-method.m
+++ b/clang/test/CodeGen/instrument-objc-method.m
@@ -11,16 +11,16 @@
 + (void)initialize {
 }
 
-// PREINLINE: declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
-// BARE: @"\01+[ObjCClass load]"{{\(.*\)}} #2
+// BARE: @"\01+[ObjCClass load]"{{\(.*\)}} #1
 + (void)load __attribute__((no_instrument_function)) {
 }
 
-// PREINLINE: @"\01-[ObjCClass dealloc]"{{\(.*\)}} #2
-// BARE: @"\01-[ObjCClass dealloc]"{{\(.*\)}} #2
+// PREINLINE: @"\01-[ObjCClass dealloc]"{{\(.*\)}} #1
+// BARE: @"\01-[ObjCClass dealloc]"{{\(.*\)}} #1
 - (void)dealloc __attribute__((no_instrument_function)) {
 }
 
+// PREINLINE: declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 // PREINLINE: attributes #0 = { {{.*}}"instrument-function-entry"="__cyg_profile_func_enter"
 // PREINLINE-NOT: attributes #0 = { {{.*}}"instrument-function-entry"="__cyg_profile_func_enter_bare"
 // PREINLINE-NOT: attributes #2 = { {{.*}}"__cyg_profile_func_enter"
diff --git a/clang/test/CodeGen/paren-list-agg-init.cpp b/clang/test/CodeGen/paren-list-agg-init.cpp
index 94d42431d125d45141c1646ef2128a5c8acc9628..88b1834d42d879191941b8ab7acd34dd0c31cdf2 100644
--- a/clang/test/CodeGen/paren-list-agg-init.cpp
+++ b/clang/test/CodeGen/paren-list-agg-init.cpp
@@ -271,14 +271,13 @@ const int* foo10() {
 // CHECK-NEXT: [[ARR_2:%.*]] = alloca [4 x i32], align 16
 // CHECK-NEXT: store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT: store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT: [[ARRINIT_BEGIN:%.*]] = getelementptr inbounds [4 x i32], ptr [[ARR_2]], i64 0, i64 0
 // CHECK-NEXT: [[TMP_0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARRINIT_BEGIN]], align 4
-// CHECK-NEXT: [[ARRINIT_ELEM:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 1
+// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARR_2]], align 4
+// CHECK-NEXT: [[ARRINIT_ELEM:%.*]] = getelementptr inbounds i32, ptr [[ARR_2]], i64 1
 // CHECK-NEXT: [[TMP_1:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK-NEXT: store i32 [[TMP_1]], ptr [[ARRINIT_ELEM]], align 4
-// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_ELEM]], i64 1
-// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 4
+// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[ARR_2]], i64 2
+// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[ARR_2]], i64 4
 // CHECK-NEXT: br label [[ARRINIT_BODY:%.*]]
 // CHECK: [[ARRINIT_CUR:%.*]] = phi ptr [ [[ARRINIT_START]], %entry ], [ [[ARRINIT_NEXT:%.*]], [[ARRINIT_BODY]] ]
 // CHECK-NEXT: store i32 0, ptr [[ARRINIT_CUR]], align 4
@@ -297,10 +296,9 @@ void foo11(int a, int b) {
 // CHECK-NEXT: [[ARR_3:%.*]] = alloca [2 x i32], align 4
 // CHECK-NEXT: store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT: store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT: [[ARRINIT_BEGIN:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARR_3]], i64 0, i64 0
 // CHECK-NEXT: [[TMP_0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARRINIT_BEGIN]], align 4
-// CHECK-NEXT: [[ARRINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 1
+// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARR_3]], align 4
+// CHECK-NEXT: [[ARRINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[ARR_3]], i64 1
 // CHECK-NEXT: [[TMP_1:%.*]] = load i32, ptr [[B_ADDR]], align 4
 // CHECK-NEXT: store i32 [[TMP_1]], ptr [[ARRINIT_ELEMENT]], align 4
 // CHECK-NEXT: ret void
@@ -336,8 +334,7 @@ const int* foo15() {
 // CHECK-NEXT: entry:
 // CHECK-NEXT: [[ARR_6:%.*arr6.*]] = alloca ptr, align 8
 // CHECK-NEXT: [[REF_TMP:%.*]] = alloca [1 x i32], align 4
-// CHECK-NEXT: [[ARRINIT_BEGIN:%.*]] = getelementptr inbounds [1 x i32], ptr [[REF_TMP]], i64 0, i64 0
-// CHECK-NEXT: store i32 3, ptr [[ARRINIT_BEGIN]], align 4
+// CHECK-NEXT: store i32 3, ptr [[REF_TMP]], align 4
 // CHECK-NEXT: store ptr [[REF_TMP]], ptr [[ARR_6]], align 8
 // CHECK-NEXT: ret void
 void foo16() {
@@ -348,10 +345,9 @@ void foo16() {
 // CHECK-NEXT: entry:
 // CHECK-NEXT: [[ARR_7:%.*arr7.*]] = alloca ptr, align 8
 // CHECK-NEXT: [[REF_TMP:%.*]] = alloca [2 x i32], align 4
-// CHECK-NEXT: [[ARRINIT_BEGIN:%.*]] = getelementptr inbounds [2 x i32], ptr [[REF_TMP]], i64 0, i64 0
-// CHECK-NEXT: store i32 4, ptr [[ARRINIT_BEGIN]], align 4
-// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 1
-// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 2
+// CHECK-NEXT: store i32 4, ptr [[REF_TMP]], align 4
+// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[REF_TMP]], i64 1
+// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[REF_TMP]], i64 2
 // CHECK-NEXT: br label [[ARRINIT_BODY]]
 // CHECK: [[ARRINIT_CUR:%.*]] = phi ptr [ [[ARRINIT_START]], %entry ], [ [[ARRINIT_NEXT:%.*]], [[ARRINIT_BODY]] ]
 // CHECK-NEXT: store i32 0, ptr [[ARRINIT_CUR]], align 4
@@ -533,14 +529,12 @@ namespace gh68198 {
   // CHECK-NEXT: entry
   // CHECK-NEXT: [[ARR_10:%.*arr9.*]] = alloca ptr, align 8
   // CHECK-NEXT: [[CALL_PTR]] = call noalias noundef nonnull ptr @_Znam(i64 noundef 16)
-  // CHECK-NEXT: [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x i32], ptr [[CALL]], i64 0, i64 0
-  // CHECK-NEXT: store i32 1, ptr [[ARRAYINIT_BEGIN]], align 4
-  // CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[ARRAYINIT_BEGIN]], i64 1
+  // CHECK-NEXT: store i32 1, ptr [[CALL]], align 4
+  // CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[CALL]], i64 1
   // CHECK-NEXT: store i32 2, ptr [[ARRAYINIT_ELEMENT]], align 4
   // CHECK-NEXT: [[ARRAY_EXP_NEXT:%.*]] = getelementptr inbounds [2 x i32], ptr %call, i64 1
-  // CHECK-NEXT: [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARRAY_EXP_NEXT]], i64 0, i64 0
-  // CHECK-NEXT: store i32 3, ptr [[ARRAYINIT_BEGIN1]], align 4
-  // CHECK-NEXT: [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds i32, ptr [[ARRAYINIT_BEGIN1]], i64 1
+  // CHECK-NEXT: store i32 3, ptr [[ARRAY_EXP_NEXT]], align 4
+  // CHECK-NEXT: [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds i32, ptr [[ARRAY_EXP_NEXT]], i64 1
   // CHECK-NEXT: store i32 4, ptr [[ARRAYINIT_ELEMENT2]], align 4
   // CHECK-NEXT: [[ARRAY_EXP_NEXT3:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARRAY_EXP_NEXT]], i64 1
   // CHECK-NEXT: store ptr [[CALL_PTR]], ptr [[ARR_10]], align 8
@@ -553,14 +547,12 @@ namespace gh68198 {
   // CHECK-NEXT: entry
   // CHECK-NEXT: [[ARR_10:%.*arr10.*]] = alloca ptr, align 8
   // CHECK-NEXT: [[CALL_PTR]] = call noalias noundef nonnull ptr @_Znam(i64 noundef 32)
-  // CHECK-NEXT: [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x i32], ptr [[CALL]], i64 0, i64 0
-  // CHECK-NEXT: store i32 5, ptr [[ARRAYINIT_BEGIN]], align 4
-  // CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[ARRAYINIT_BEGIN]], i64 1
+  // CHECK-NEXT: store i32 5, ptr [[CALL]], align 4
+  // CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds i32, ptr [[CALL]], i64 1
   // CHECK-NEXT: store i32 6, ptr [[ARRAYINIT_ELEMENT]], align 4
   // CHECK-NEXT: [[ARRAY_EXP_NEXT:%.*]] = getelementptr inbounds [2 x i32], ptr %call, i64 1
-  // CHECK-NEXT: [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARRAY_EXP_NEXT]], i64 0, i64 0
-  // CHECK-NEXT: store i32 7, ptr [[ARRAYINIT_BEGIN1]], align 4
-  // CHECK-NEXT: [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds i32, ptr [[ARRAYINIT_BEGIN1]], i64 1
+  // CHECK-NEXT: store i32 7, ptr [[ARRAY_EXP_NEXT]], align 4
+  // CHECK-NEXT: [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds i32, ptr [[ARRAY_EXP_NEXT]], i64 1
   // CHECK-NEXT: store i32 8, ptr [[ARRAYINIT_ELEMENT2]], align 4
   // CHECK-NEXT: [[ARRAY_EXP_NEXT3:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARRAY_EXP_NEXT]], i64 1
   // CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[ARRAY_EXP_NEXT3]], i8 0, i64 16, i1 false)
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 9d86880d6513e036d199ce5e48552fbc243c683b..7f7005d21b99a3bd9ef043f56d5dbc5e7a768bbf 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -268,3 +268,7 @@
 // RUN: %clang_cc1 -triple ve -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=VE
 // VE: target datalayout = "e-m:e-i64:64-n32:64-S128-v64:64:64-v128:64:64-v256:64:64-v512:64:64-v1024:64:64-v2048:64:64-v4096:64:64-v8192:64:64-v16384:64:64"
+
+// RUN: %clang_cc1 -triple spirv64-amd -o - -emit-llvm %s | \
+// RUN: FileCheck %s -check-prefix=SPIR64
+// AMDGPUSPIRV64: target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0"
diff --git a/clang/test/CodeGen/voidptr-vaarg.c b/clang/test/CodeGen/voidptr-vaarg.c
new file mode 100644
index 0000000000000000000000000000000000000000..d023ddf0fb5d2d2aa6fde6c06d9a29f5bc4a3f54
--- /dev/null
+++ b/clang/test/CodeGen/voidptr-vaarg.c
@@ -0,0 +1,478 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+
+// Multiple targets use emitVoidPtrVAArg to lower va_arg instructions in clang
+// PPC is complicated, excluding from this case analysis
+// ForceRightAdjust is false for all non-PPC targets
+// AllowHigherAlign is only false for two Microsoft targets, both of which
+// pass most things by reference.
+//
+// Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
+//                          QualType ValueTy, bool IsIndirect,
+//                          TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign,
+//                          bool AllowHigherAlign, bool ForceRightAdjust =
+//                          false);
+//
+// Target       IsIndirect    SlotSize  AllowHigher ForceRightAdjust
+// ARC          false             four  true        false
+// ARM          varies            four  true        false
+// Mips         false           4 or 8  true        false
+// RISCV        varies        register  true        false
+// PPC elided
+// LoongArch    varies        register  true        false
+// NVPTX WIP
+// AMDGPU WIP
+// X86_32       false             four  true        false
+// X86_64 MS    varies           eight  false       false
+// CSKY         false             four  true        false
+// Webassembly  varies            four  true        false
+// AArch64      false            eight  true        false
+// AArch64 MS   false            eight  false       false
+//
+// Webassembly passes indirectly iff it's an aggregate of multiple values
+// Choosing this as a representative architecture to check IR generation
+// partly because it has a relatively simple variadic calling convention.
+
+// Int, by itself and packed in structs
+// CHECK-LABEL: @raw_int(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int raw_int(__builtin_va_list list) { return __builtin_va_arg(list, int); }
+
+typedef struct {
+  int x;
+} one_int_t;
+
+// CHECK-LABEL: @one_int(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_ONE_INT_T:%.*]], align 4
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false)
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_INT_T]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COERCE_DIVE]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+one_int_t one_int(__builtin_va_list list) {
+  return __builtin_va_arg(list, one_int_t);
+}
+
+typedef struct {
+  int x;
+  int y;
+} two_int_t;
+
+// CHECK-LABEL: @two_int(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_RESULT:%.*]], ptr align 4 [[TMP0]], i32 8, i1 false)
+// CHECK-NEXT:    ret void
+//
+two_int_t two_int(__builtin_va_list list) {
+  return __builtin_va_arg(list, two_int_t);
+}
+
+// Double, by itself and packed in structs
+// CHECK-LABEL: @raw_double(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
+// CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8
+// CHECK-NEXT:    ret double [[TMP1]]
+//
+double raw_double(__builtin_va_list list) {
+  return __builtin_va_arg(list, double);
+}
+
+typedef struct {
+  double x;
+} one_double_t;
+
+// CHECK-LABEL: @one_double(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_ONE_DOUBLE_T:%.*]], align 8
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
+// CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[RETVAL]], ptr align 8 [[ARGP_CUR_ALIGNED]], i32 8, i1 false)
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_DOUBLE_T]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[COERCE_DIVE]], align 8
+// CHECK-NEXT:    ret double [[TMP1]]
+//
+one_double_t one_double(__builtin_va_list list) {
+  return __builtin_va_arg(list, one_double_t);
+}
+
+typedef struct {
+  double x;
+  double y;
+} two_double_t;
+
+// CHECK-LABEL: @two_double(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[AGG_RESULT:%.*]], ptr align 8 [[TMP0]], i32 16, i1 false)
+// CHECK-NEXT:    ret void
+//
+two_double_t two_double(__builtin_va_list list) {
+  return __builtin_va_arg(list, two_double_t);
+}
+
+// Scalar smaller than the slot size (C would promote a short to int)
+typedef struct {
+  char x;
+} one_char_t;
+
+// CHECK-LABEL: @one_char(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_ONE_CHAR_T:%.*]], align 1
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 1, i1 false)
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_CHAR_T]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[COERCE_DIVE]], align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+one_char_t one_char(__builtin_va_list list) {
+  return __builtin_va_arg(list, one_char_t);
+}
+
+typedef struct {
+  short x;
+} one_short_t;
+
+// CHECK-LABEL: @one_short(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_ONE_SHORT_T:%.*]], align 2
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 2, i1 false)
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_SHORT_T]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[COERCE_DIVE]], align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+one_short_t one_short(__builtin_va_list list) {
+  return __builtin_va_arg(list, one_short_t);
+}
+
+// Composite smaller than the slot size
+typedef struct {
+  _Alignas(2) char x;
+  char y;
+} char_pair_t;
+
+// CHECK-LABEL: @char_pair(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[AGG_RESULT:%.*]], ptr align 2 [[TMP0]], i32 2, i1 false)
+// CHECK-NEXT:    ret void
+//
+char_pair_t char_pair(__builtin_va_list list) {
+  return __builtin_va_arg(list, char_pair_t);
+}
+
+// Empty struct
+typedef struct {
+} empty_t;
+
+// CHECK-LABEL: @empty(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY_T:%.*]], align 1
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 0
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 0, i1 false)
+// CHECK-NEXT:    ret void
+//
+empty_t empty(__builtin_va_list list) {
+  return __builtin_va_arg(list, empty_t);
+}
+
+typedef struct {
+  empty_t x;
+  int y;
+} empty_int_t;
+
+// CHECK-LABEL: @empty_int(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY_INT_T:%.*]], align 4
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false)
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[RETVAL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+empty_int_t empty_int(__builtin_va_list list) {
+  return __builtin_va_arg(list, empty_int_t);
+}
+
+typedef struct {
+  int x;
+  empty_t y;
+} int_empty_t;
+
+// CHECK-LABEL: @int_empty(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_INT_EMPTY_T:%.*]], align 4
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false)
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_INT_EMPTY_T]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COERCE_DIVE]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int_empty_t int_empty(__builtin_va_list list) {
+  return __builtin_va_arg(list, int_empty_t);
+}
+
+// Need multiple va_arg instructions to check the postincrement
+// Using types that are passed directly as the indirect handling
+// is independent of the alignment handling in emitVoidPtrDirectVAArg.
+
+// CHECK-LABEL: @multiple_int(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT0_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT1_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT2_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP1]], align 4
+// CHECK-NEXT:    [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARGP_CUR1]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
+// CHECK-NEXT:    [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARGP_CUR3]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP5]], align 4
+// CHECK-NEXT:    ret void
+//
+void multiple_int(__builtin_va_list list, int *out0, int *out1, int *out2) {
+  *out0 = __builtin_va_arg(list, int);
+  *out1 = __builtin_va_arg(list, int);
+  *out2 = __builtin_va_arg(list, int);
+}
+
+// Scalars in structs are an easy way of specifying alignment from C
+// CHECK-LABEL: @increasing_alignment(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT0_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT1_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT2_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT3_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT3:%.*]], ptr [[OUT3_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TMP0]], ptr align 4 [[ARGP_CUR]], i32 1, i1 false)
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[TMP1]], ptr align 4 [[ARGP_CUR1]], i32 2, i1 false)
+// CHECK-NEXT:    [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARGP_CUR3]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
+// CHECK-NEXT:    [[ARGP_CUR5:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5]], i32 7
+// CHECK-NEXT:    [[ARGP_CUR5_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP4]], i32 -8)
+// CHECK-NEXT:    [[ARGP_NEXT6:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5_ALIGNED]], i32 8
+// CHECK-NEXT:    store ptr [[ARGP_NEXT6]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARGP_CUR5_ALIGNED]], align 8
+// CHECK-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[OUT3_ADDR]], align 4
+// CHECK-NEXT:    store double [[TMP5]], ptr [[TMP6]], align 8
+// CHECK-NEXT:    ret void
+//
+void increasing_alignment(__builtin_va_list list, one_char_t *out0,
+                          one_short_t *out1, int *out2, double *out3) {
+  *out0 = __builtin_va_arg(list, one_char_t);
+  *out1 = __builtin_va_arg(list, one_short_t);
+  *out2 = __builtin_va_arg(list, int);
+  *out3 = __builtin_va_arg(list, double);
+}
+
+// CHECK-LABEL: @decreasing_alignment(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT0_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT1_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT2_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT3_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT3:%.*]], ptr [[OUT3_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
+// CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4
+// CHECK-NEXT:    store double [[TMP1]], ptr [[TMP2]], align 8
+// CHECK-NEXT:    [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARGP_CUR1]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[TMP5]], ptr align 4 [[ARGP_CUR3]], i32 2, i1 false)
+// CHECK-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[OUT3_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR5:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT6:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT6]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TMP6]], ptr align 4 [[ARGP_CUR5]], i32 1, i1 false)
+// CHECK-NEXT:    ret void
+//
+void decreasing_alignment(__builtin_va_list list, double *out0, int *out1,
+                          one_short_t *out2, one_char_t *out3) {
+  *out0 = __builtin_va_arg(list, double);
+  *out1 = __builtin_va_arg(list, int);
+  *out2 = __builtin_va_arg(list, one_short_t);
+  *out3 = __builtin_va_arg(list, one_char_t);
+}
+
+// Typical edge cases, none hit special handling in VAArg lowering.
+typedef struct {
+  int x[16];
+  double y[8];
+} large_value_t;
+
+// CHECK-LABEL: @large_value(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[TMP0]], ptr align 8 [[TMP1]], i32 128, i1 false)
+// CHECK-NEXT:    ret void
+//
+void large_value(__builtin_va_list list, large_value_t *out) {
+  *out = __builtin_va_arg(list, large_value_t);
+}
+
+typedef int v128_t __attribute__((__vector_size__(16), __aligned__(16)));
+// CHECK-LABEL: @vector(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 15
+// CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -16)
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 16
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[ARGP_CUR_ALIGNED]], align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[TMP2]], align 16
+// CHECK-NEXT:    ret void
+//
+void vector(__builtin_va_list list, v128_t *out) {
+  *out = __builtin_va_arg(list, v128_t);
+}
+
+typedef struct BF {
+  float not_an_i32[2];
+  int A : 1;
+  char B;
+  int C : 13;
+} BF;
+
+// CHECK-LABEL: @bitfield(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[LIST_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr, align 4
+// CHECK-NEXT:    store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
+// CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP0]], ptr align 4 [[TMP1]], i32 12, i1 false)
+// CHECK-NEXT:    ret void
+//
+void bitfield(__builtin_va_list list, BF *out) {
+  *out = __builtin_va_arg(list, BF);
+}
diff --git a/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu b/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu
new file mode 100644
index 0000000000000000000000000000000000000000..8dbb8c538ddc16b9b583a8084bc761ccd45dd7c9
--- /dev/null
+++ b/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu
@@ -0,0 +1,294 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \
+// RUN:  -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -emit-llvm %s \
+// RUN:  -o - | FileCheck %s
+
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \
+// RUN:  -aux-triple x86_64-pc-windows-msvc -fcuda-is-device -emit-llvm %s \
+// RUN:  -o - | FileCheck %s
+
+#include "Inputs/cuda.h"
+
+// CHECK-LABEL: @_Z16use_dispatch_ptrPi(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[DISPATCH_PTR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[DISPATCH_PTR_ASCAST:%.*]] = addrspacecast ptr [[DISPATCH_PTR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call align 4 dereferenceable(64) addrspace(4) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[DISPATCH_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[DISPATCH_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4
+// CHECK-NEXT:    ret void
+//
+__global__ void use_dispatch_ptr(int* out) {
+  const int* dispatch_ptr = (const int*)__builtin_amdgcn_dispatch_ptr();
+  *out = *dispatch_ptr;
+}
+
+// CHECK-LABEL: @_Z13use_queue_ptrPi(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[QUEUE_PTR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[QUEUE_PTR_ASCAST:%.*]] = addrspacecast ptr [[QUEUE_PTR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call addrspace(4) ptr addrspace(4) @llvm.amdgcn.queue.ptr()
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[QUEUE_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[QUEUE_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4
+// CHECK-NEXT:    ret void
+//
+__global__ void use_queue_ptr(int* out) {
+  const int* queue_ptr = (const int*)__builtin_amdgcn_queue_ptr();
+  *out = *queue_ptr;
+}
+
+// CHECK-LABEL: @_Z19use_implicitarg_ptrPi(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[IMPLICITARG_PTR_ASCAST:%.*]] = addrspacecast ptr [[IMPLICITARG_PTR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call addrspace(4) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[IMPLICITARG_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[IMPLICITARG_PTR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4
+// CHECK-NEXT:    ret void
+//
+__global__ void use_implicitarg_ptr(int* out) {
+  const int* implicitarg_ptr = (const int*)__builtin_amdgcn_implicitarg_ptr();
+  *out = *implicitarg_ptr;
+}
+
+__global__
+    //
+    void
+// CHECK-LABEL: @_Z12test_ds_fmaxf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SRC_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) @_ZZ12test_ds_fmaxfE6shared, float [[TMP0]], i32 0, i32 0, i1 false)
+// CHECK-NEXT:    store volatile float [[TMP1]], ptr addrspace(4) [[X_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+    test_ds_fmax(float src) {
+  __shared__ float shared;
+  volatile float x = __builtin_amdgcn_ds_fmaxf(&shared, src, 0, 0, false);
+}
+
+// CHECK-LABEL: @_Z12test_ds_faddf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SRC_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) @_ZZ12test_ds_faddfE6shared, float [[TMP0]], i32 0, i32 0, i1 false)
+// CHECK-NEXT:    store volatile float [[TMP1]], ptr addrspace(4) [[X_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__global__ void test_ds_fadd(float src) {
+  __shared__ float shared;
+  volatile float x = __builtin_amdgcn_ds_faddf(&shared, src, 0, 0, false);
+}
+
+// CHECK-LABEL: @_Z12test_ds_fminfPf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SHARED:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[SRC_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SHARED_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SHARED_ASCAST:%.*]] = addrspacecast ptr [[SHARED]] to ptr addrspace(4)
+// CHECK-NEXT:    [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[SHARED_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SHARED_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[SHARED_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[SHARED_ASCAST]], align 8
+// CHECK-NEXT:    [[SHARED1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ASCAST]], align 8
+// CHECK-NEXT:    store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    store ptr addrspace(4) [[SHARED1]], ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr addrspace(3)
+// CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) [[TMP2]], float [[TMP3]], i32 0, i32 0, i1 false)
+// CHECK-NEXT:    store volatile float [[TMP4]], ptr addrspace(4) [[X_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__global__ void test_ds_fmin(float src, float *shared) {
+  volatile float x = __builtin_amdgcn_ds_fminf(shared, src, 0, 0, false);
+}
+
+#if 0 // FIXME: returning a pointer to AS4 explicitly is wrong for AMDGPU SPIRV
+//
+__device__ void test_ret_builtin_nondef_addrspace() {
+  void *x = __builtin_amdgcn_dispatch_ptr();
+}
+#endif
+
+// CHECK-LABEL: @_Z6endpgmv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call addrspace(4) void @llvm.amdgcn.endpgm()
+// CHECK-NEXT:    ret void
+//
+__global__ void endpgm() {
+  __builtin_amdgcn_endpgm();
+}
+
+// Check the 64 bit argument is correctly passed to the intrinsic without truncation or assertion.
+
+// CHECK-LABEL: @_Z14test_uicmp_i64Pyyy(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i64 [[A:%.*]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i64 [[B:%.*]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr addrspace(4) [[B_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = call addrspace(4) i64 @llvm.amdgcn.icmp.i64.i64(i64 [[TMP1]], i64 [[TMP2]], i32 35)
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i64 [[TMP3]], ptr addrspace(4) [[TMP4]], align 8
+// CHECK-NEXT:    ret void
+//
+__global__ void test_uicmp_i64(unsigned long long *out, unsigned long long a, unsigned long long b)
+{
+  *out = __builtin_amdgcn_uicmpl(a, b, 30+5);
+}
+
+// Check the 64 bit return value is correctly returned without truncation or assertion.
+
+// CHECK-LABEL: @_Z14test_s_memtimePy(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call addrspace(4) i64 @llvm.amdgcn.s.memtime()
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(4) [[TMP2]], align 8
+// CHECK-NEXT:    ret void
+//
+__global__ void test_s_memtime(unsigned long long* out)
+{
+  *out = __builtin_amdgcn_s_memtime();
+}
+
+// Check a generic pointer can be passed as a shared pointer and a generic pointer.
+__device__ void func(float *x);
+
+// CHECK-LABEL: @_Z17test_ds_fmin_funcfPf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SHARED:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[SRC_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SHARED_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[SHARED_ASCAST:%.*]] = addrspacecast ptr [[SHARED]] to ptr addrspace(4)
+// CHECK-NEXT:    [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[SHARED_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SHARED_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[SHARED_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[SHARED_ASCAST]], align 8
+// CHECK-NEXT:    [[SHARED1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ASCAST]], align 8
+// CHECK-NEXT:    store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    store ptr addrspace(4) [[SHARED1]], ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr addrspace(3)
+// CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) [[TMP2]], float [[TMP3]], i32 0, i32 0, i1 false)
+// CHECK-NEXT:    store volatile float [[TMP4]], ptr addrspace(4) [[X_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    call spir_func addrspace(4) void @_Z4funcPf(ptr addrspace(4) noundef [[TMP5]]) #[[ATTR7:[0-9]+]]
+// CHECK-NEXT:    ret void
+//
+__global__ void test_ds_fmin_func(float src, float *__restrict shared) {
+  volatile float x = __builtin_amdgcn_ds_fminf(shared, src, 0, 0, false);
+  func(shared);
+}
+
+// CHECK-LABEL: @_Z14test_is_sharedPf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[RET:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ADDR_ASCAST:%.*]] = addrspacecast ptr [[X_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[RET_ASCAST:%.*]] = addrspacecast ptr [[RET]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[X_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[X_ASCAST]], align 8
+// CHECK-NEXT:    [[X1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[X1]], ptr addrspace(4) [[X_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr
+// CHECK-NEXT:    [[TMP3:%.*]] = call addrspace(4) i1 @llvm.amdgcn.is.shared(ptr [[TMP2]])
+// CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8
+// CHECK-NEXT:    store i8 [[FROMBOOL]], ptr addrspace(4) [[RET_ASCAST]], align 1
+// CHECK-NEXT:    ret void
+//
+__global__ void test_is_shared(float *x){
+  bool ret = __builtin_amdgcn_is_shared(x);
+}
+
+// CHECK-LABEL: @_Z15test_is_privatePi(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[RET:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    [[X_ADDR_ASCAST:%.*]] = addrspacecast ptr [[X_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[RET_ASCAST:%.*]] = addrspacecast ptr [[RET]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[X_COERCE:%.*]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[X_ASCAST]], align 8
+// CHECK-NEXT:    [[X1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[X1]], ptr addrspace(4) [[X_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr
+// CHECK-NEXT:    [[TMP3:%.*]] = call addrspace(4) i1 @llvm.amdgcn.is.private(ptr [[TMP2]])
+// CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8
+// CHECK-NEXT:    store i8 [[FROMBOOL]], ptr addrspace(4) [[RET_ASCAST]], align 1
+// CHECK-NEXT:    ret void
+//
+__global__ void test_is_private(int *x){
+  bool ret = __builtin_amdgcn_is_private(x);
+}
diff --git a/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu b/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu
new file mode 100644
index 0000000000000000000000000000000000000000..1ea1d5f454762dbab51ce4238dc0ac44954a0053
--- /dev/null
+++ b/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu
@@ -0,0 +1,31 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \
+// RUN:  -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -emit-llvm %s \
+// RUN:  -o - | FileCheck %s
+
+#define __device__ __attribute__((device))
+typedef __attribute__((address_space(3))) float *LP;
+
+// CHECK-LABEL: define spir_func void @_Z22test_ds_atomic_add_f32Pff(
+// CHECK-SAME: ptr addrspace(4) noundef [[ADDR:%.*]], float noundef [[VAL:%.*]]) addrspace(4) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADDR_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[RTN:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[ADDR_ADDR_ASCAST:%.*]] = addrspacecast ptr [[ADDR_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr [[VAL_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[RTN_ASCAST:%.*]] = addrspacecast ptr [[RTN]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[ADDR]], ptr addrspace(4) [[ADDR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store float [[VAL]], ptr addrspace(4) [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ADDR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr addrspace(3)
+// CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr addrspace(4) [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) [[TMP1]], float [[TMP2]], i32 0, i32 0, i1 false)
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[RTN_ASCAST]], align 8
+// CHECK-NEXT:    store float [[TMP3]], ptr addrspace(4) [[TMP4]], align 4
+// CHECK-NEXT:    ret void
+//
+__device__ void test_ds_atomic_add_f32(float *addr, float val) {
+  float *rtn;
+  *rtn = __builtin_amdgcn_ds_faddf((LP)addr, val, 0, 0, 0);
+}
diff --git a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu
index ba5e5f13ebe707f278faef55deda994f693c319b..7880a8036f8cd405c20f47b8748863c7dc549152 100644
--- a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu
+++ b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu
@@ -6,21 +6,21 @@
 __attribute__((global))
 void kernel(int *out) {
   int i = 0;
-  out[i++] = threadIdx.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.x()
-  out[i++] = threadIdx.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.y()
-  out[i++] = threadIdx.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+  out[i++] = threadIdx.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  out[i++] = threadIdx.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+  out[i++] = threadIdx.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.z()
 
-  out[i++] = blockIdx.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
-  out[i++] = blockIdx.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
-  out[i++] = blockIdx.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
+  out[i++] = blockIdx.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
+  out[i++] = blockIdx.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
+  out[i++] = blockIdx.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
 
-  out[i++] = blockDim.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
-  out[i++] = blockDim.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
-  out[i++] = blockDim.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+  out[i++] = blockDim.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+  out[i++] = blockDim.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+  out[i++] = blockDim.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
 
-  out[i++] = gridDim.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
-  out[i++] = gridDim.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
-  out[i++] = gridDim.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
+  out[i++] = gridDim.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+  out[i++] = gridDim.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
+  out[i++] = gridDim.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
 
   out[i++] = warpSize; // CHECK: store i32 32,
 
diff --git a/clang/test/CodeGenCUDA/long-double.cu b/clang/test/CodeGenCUDA/long-double.cu
index d52de972ea3da4e9aaf4c2fed0e4423ae003ec0a..898afcac124b5f841f0d11f08eb301994387362a 100644
--- a/clang/test/CodeGenCUDA/long-double.cu
+++ b/clang/test/CodeGenCUDA/long-double.cu
@@ -2,6 +2,10 @@
 // RUN:   -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \
 // RUN:   -emit-llvm -o - -x hip %s 2>&1 | FileCheck %s
 
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa \
+// RUN:   -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \
+// RUN:   -emit-llvm -o - -x hip %s 2>&1 | FileCheck %s
+
 // RUN: %clang_cc1 -triple nvptx \
 // RUN:   -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \
 // RUN:   -emit-llvm -o - %s 2>&1 | FileCheck %s
diff --git a/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu b/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu
new file mode 100644
index 0000000000000000000000000000000000000000..2a0f84d1daa758d1b9ee105420f5068a6b0af2cb
--- /dev/null
+++ b/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu
@@ -0,0 +1,129 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// REQUIRES: x86-registered-target
+
+// RUN: %clang_cc1 "-aux-triple" "x86_64-unknown-linux-gnu" "-triple" "spirv64-amd-amdhsa" \
+// RUN:    -fcuda-is-device "-aux-target-cpu" "x86-64" -emit-llvm -o - %s | FileCheck %s
+
+#include "Inputs/cuda.h"
+
+// CHECK-LABEL: @_Z8test_argPDF16bDF16b(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[IN_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[BF16:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[BF16_ASCAST:%.*]] = addrspacecast ptr [[BF16]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT:%.*]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    store bfloat [[TMP0]], ptr addrspace(4) [[BF16_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load bfloat, ptr addrspace(4) [[BF16_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr addrspace(4) [[TMP2]], align 2
+// CHECK-NEXT:    ret void
+//
+__device__ void test_arg(__bf16 *out, __bf16 in) {
+  __bf16 bf16 = in;
+  *out = bf16;
+}
+
+// CHECK-LABEL: @_Z9test_loadPDF16bS_(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[IN_ADDR:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[BF16:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[BF16_ASCAST:%.*]] = addrspacecast ptr [[BF16]] to ptr addrspace(4)
+// CHECK-NEXT:    store ptr addrspace(4) [[OUT:%.*]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store ptr addrspace(4) [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[IN_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load bfloat, ptr addrspace(4) [[TMP0]], align 2
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr addrspace(4) [[BF16_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = load bfloat, ptr addrspace(4) [[BF16_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store bfloat [[TMP2]], ptr addrspace(4) [[TMP3]], align 2
+// CHECK-NEXT:    ret void
+//
+__device__ void test_load(__bf16 *out, __bf16 *in) {
+  __bf16 bf16 = *in;
+  *out = bf16;
+}
+
+// CHECK-LABEL: @_Z8test_retDF16b(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[IN_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
+// CHECK-NEXT:    [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    ret bfloat [[TMP0]]
+//
+__device__ __bf16 test_ret( __bf16 in) {
+  return in;
+}
+
+// CHECK-LABEL: @_Z9test_callDF16b(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[IN_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
+// CHECK-NEXT:    [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[CALL:%.*]] = call contract spir_func noundef addrspace(4) bfloat @_Z8test_retDF16b(bfloat noundef [[TMP0]]) #[[ATTR1:[0-9]+]]
+// CHECK-NEXT:    ret bfloat [[CALL]]
+//
+__device__ __bf16 test_call( __bf16 in) {
+  return test_ret(in);
+}
+
+
+// CHECK-LABEL: @_Z15test_vec_assignv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VEC2_A:%.*]] = alloca <2 x bfloat>, align 4
+// CHECK-NEXT:    [[VEC2_B:%.*]] = alloca <2 x bfloat>, align 4
+// CHECK-NEXT:    [[VEC4_A:%.*]] = alloca <4 x bfloat>, align 8
+// CHECK-NEXT:    [[VEC4_B:%.*]] = alloca <4 x bfloat>, align 8
+// CHECK-NEXT:    [[VEC8_A:%.*]] = alloca <8 x bfloat>, align 16
+// CHECK-NEXT:    [[VEC8_B:%.*]] = alloca <8 x bfloat>, align 16
+// CHECK-NEXT:    [[VEC16_A:%.*]] = alloca <16 x bfloat>, align 32
+// CHECK-NEXT:    [[VEC16_B:%.*]] = alloca <16 x bfloat>, align 32
+// CHECK-NEXT:    [[VEC2_A_ASCAST:%.*]] = addrspacecast ptr [[VEC2_A]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC2_B_ASCAST:%.*]] = addrspacecast ptr [[VEC2_B]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC4_A_ASCAST:%.*]] = addrspacecast ptr [[VEC4_A]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC4_B_ASCAST:%.*]] = addrspacecast ptr [[VEC4_B]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC8_A_ASCAST:%.*]] = addrspacecast ptr [[VEC8_A]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC8_B_ASCAST:%.*]] = addrspacecast ptr [[VEC8_B]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC16_A_ASCAST:%.*]] = addrspacecast ptr [[VEC16_A]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VEC16_B_ASCAST:%.*]] = addrspacecast ptr [[VEC16_B]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x bfloat>, ptr addrspace(4) [[VEC2_B_ASCAST]], align 4
+// CHECK-NEXT:    store <2 x bfloat> [[TMP0]], ptr addrspace(4) [[VEC2_A_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x bfloat>, ptr addrspace(4) [[VEC4_B_ASCAST]], align 8
+// CHECK-NEXT:    store <4 x bfloat> [[TMP1]], ptr addrspace(4) [[VEC4_A_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <8 x bfloat>, ptr addrspace(4) [[VEC8_B_ASCAST]], align 16
+// CHECK-NEXT:    store <8 x bfloat> [[TMP2]], ptr addrspace(4) [[VEC8_A_ASCAST]], align 16
+// CHECK-NEXT:    [[TMP3:%.*]] = load <16 x bfloat>, ptr addrspace(4) [[VEC16_B_ASCAST]], align 32
+// CHECK-NEXT:    store <16 x bfloat> [[TMP3]], ptr addrspace(4) [[VEC16_A_ASCAST]], align 32
+// CHECK-NEXT:    ret void
+//
+__device__ void test_vec_assign() {
+  typedef __attribute__((ext_vector_type(2))) __bf16 bf16_x2;
+  bf16_x2 vec2_a, vec2_b;
+  vec2_a = vec2_b;
+
+  typedef __attribute__((ext_vector_type(4))) __bf16 bf16_x4;
+  bf16_x4 vec4_a, vec4_b;
+  vec4_a = vec4_b;
+
+  typedef __attribute__((ext_vector_type(8))) __bf16 bf16_x8;
+  bf16_x8 vec8_a, vec8_b;
+  vec8_a = vec8_b;
+
+  typedef __attribute__((ext_vector_type(16))) __bf16 bf16_x16;
+  bf16_x16 vec16_a, vec16_b;
+  vec16_a = vec16_b;
+}
diff --git a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
index ac466ee5bba40b0dd2d24aa0159e115611669898..4eafa720e0cb41b0248835a9475ae2cbb08399a5 100644
--- a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
+++ b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
@@ -158,16 +158,15 @@ void ArrayInit() {
   // CHECK-LABEL: define dso_local void @_Z9ArrayInitv()
   // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
   // CHECK: %cleanup.dest.slot = alloca i32, align 4
-  // CHECK: %arrayinit.begin = getelementptr inbounds [4 x %struct.Printy], ptr %arr, i64 0, i64 0
-  // CHECK: store ptr %arrayinit.begin, ptr %arrayinit.endOfInit, align 8
+  // CHECK: store ptr %arr, ptr %arrayinit.endOfInit, align 8
   Printy arr[4] = {
     Printy("a"),
-    // CHECK: call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arrayinit.begin, ptr noundef @.str)
-    // CHECK: [[ARRAYINIT_ELEMENT1:%.+]] = getelementptr inbounds %struct.Printy, ptr %arrayinit.begin, i64 1
+    // CHECK: call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arr, ptr noundef @.str)
+    // CHECK: [[ARRAYINIT_ELEMENT1:%.+]] = getelementptr inbounds %struct.Printy, ptr %arr, i64 1
     // CHECK: store ptr [[ARRAYINIT_ELEMENT1]], ptr %arrayinit.endOfInit, align 8
     Printy("b"),
     // CHECK: call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT1]], ptr noundef @.str.1)
-    // CHECK: [[ARRAYINIT_ELEMENT2:%.+]] = getelementptr inbounds %struct.Printy, ptr [[ARRAYINIT_ELEMENT1]], i64 1
+    // CHECK: [[ARRAYINIT_ELEMENT2:%.+]] = getelementptr inbounds %struct.Printy, ptr %arr, i64 2
     // CHECK: store ptr [[ARRAYINIT_ELEMENT2]], ptr %arrayinit.endOfInit, align 8
     ({
     // CHECK: br i1 {{.*}}, label %if.then, label %if.end
@@ -180,7 +179,7 @@ void ArrayInit() {
       // CHECK:       if.end:
       Printy("c");
       // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-      // CHECK-NEXT:    %arrayinit.element2 = getelementptr inbounds %struct.Printy, ptr %arrayinit.element1, i64 1
+      // CHECK-NEXT:    %arrayinit.element2 = getelementptr inbounds %struct.Printy, ptr %arr, i64 3
       // CHECK-NEXT:    store ptr %arrayinit.element2, ptr %arrayinit.endOfInit, align 8
     }),
     ({
@@ -212,14 +211,14 @@ void ArrayInit() {
 
   // CHECK:       cleanup:
   // CHECK-NEXT:    %1 = load ptr, ptr %arrayinit.endOfInit, align 8
-  // CHECK-NEXT:    %arraydestroy.isempty = icmp eq ptr %arrayinit.begin, %1
+  // CHECK-NEXT:    %arraydestroy.isempty = icmp eq ptr %arr, %1
   // CHECK-NEXT:    br i1 %arraydestroy.isempty, label %[[ARRAY_DESTROY_DONE2:.+]], label %[[ARRAY_DESTROY_BODY2:.+]]
 
   // CHECK:       [[ARRAY_DESTROY_BODY2]]:
   // CHECK-NEXT:    %arraydestroy.elementPast = phi ptr [ %1, %cleanup ], [ %arraydestroy.element, %[[ARRAY_DESTROY_BODY2]] ]
   // CHECK-NEXT:    %arraydestroy.element = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast, i64 -1
   // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element)
-  // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, %arrayinit.begin
+  // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, %arr
   // CHECK-NEXT:    br i1 %arraydestroy.done, label %[[ARRAY_DESTROY_DONE2]], label %[[ARRAY_DESTROY_BODY2]]
 
   // CHECK:       [[ARRAY_DESTROY_DONE2]]:
@@ -238,8 +237,7 @@ void ArraySubobjects() {
       // CHECK: call void @_ZN6PrintyC1EPKc
       // CHECK: call void @_ZN6PrintyC1EPKc
       {Printy("a"),
-      // CHECK: [[ARRAYINIT_BEGIN:%.+]] = getelementptr inbounds [2 x %struct.Printy]
-      // CHECK: store ptr [[ARRAYINIT_BEGIN]], ptr %arrayinit.endOfInit, align 8
+      // CHECK: store ptr %arr2, ptr %arrayinit.endOfInit, align 8
       // CHECK: call void @_ZN6PrintyC1EPKc
       // CHECK: [[ARRAYINIT_ELEMENT:%.+]] = getelementptr inbounds %struct.Printy
       // CHECK: store ptr [[ARRAYINIT_ELEMENT]], ptr %arrayinit.endOfInit, align 8
@@ -248,7 +246,7 @@ void ArraySubobjects() {
            return;
            // CHECK:      if.then:
            // CHECK-NEXT:   [[V0:%.+]] = load ptr, ptr %arrayinit.endOfInit, align 8
-           // CHECK-NEXT:   %arraydestroy.isempty = icmp eq ptr [[ARRAYINIT_BEGIN]], [[V0]]
+           // CHECK-NEXT:   %arraydestroy.isempty = icmp eq ptr %arr2, [[V0]]
            // CHECK-NEXT:   br i1 %arraydestroy.isempty, label %[[ARRAY_DESTROY_DONE:.+]], label %[[ARRAY_DESTROY_BODY:.+]]
          }
          Printy("b");
@@ -268,7 +266,7 @@ void ArraySubobjects() {
     // CHECK-NEXT:    %arraydestroy.elementPast = phi ptr [ %0, %if.then ], [ %arraydestroy.element, %[[ARRAY_DESTROY_BODY]] ]
     // CHECK-NEXT:    %arraydestroy.element = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast, i64 -1
     // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element)
-    // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, [[ARRAYINIT_BEGIN]]
+    // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, %arr2
     // CHECK-NEXT:    br i1 %arraydestroy.done, label %[[ARRAY_DESTROY_DONE]], label %[[ARRAY_DESTROY_BODY]]
 
     // CHECK:       [[ARRAY_DESTROY_DONE]]
@@ -277,11 +275,11 @@ void ArraySubobjects() {
     // CHECK-NEXT:    br label %[[ARRAY_DESTROY_BODY2:.+]]
 
     // CHECK:       [[ARRAY_DESTROY_BODY2]]:
-    // CHECK-NEXT:    %arraydestroy.elementPast5 = phi ptr [ %1, %[[ARRAY_DESTROY_DONE]] ], [ %arraydestroy.element6, %[[ARRAY_DESTROY_BODY2]] ]
-    // CHECK-NEXT:    %arraydestroy.element6 = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast5, i64 -1
-    // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element6)
-    // CHECK-NEXT:    %arraydestroy.done7 = icmp eq ptr %arraydestroy.element6, [[ARRAY_BEGIN]]
-    // CHECK-NEXT:    br i1 %arraydestroy.done7, label %[[ARRAY_DESTROY_DONE2:.+]], label %[[ARRAY_DESTROY_BODY2]]
+    // CHECK-NEXT:    %arraydestroy.elementPast4 = phi ptr [ %1, %[[ARRAY_DESTROY_DONE]] ], [ %arraydestroy.element5, %[[ARRAY_DESTROY_BODY2]] ]
+    // CHECK-NEXT:    %arraydestroy.element5 = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast4, i64 -1
+    // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element5)
+    // CHECK-NEXT:    %arraydestroy.done6 = icmp eq ptr %arraydestroy.element5, [[ARRAY_BEGIN]]
+    // CHECK-NEXT:    br i1 %arraydestroy.done6, label %[[ARRAY_DESTROY_DONE2:.+]], label %[[ARRAY_DESTROY_BODY2]]
 
 
     // CHECK:     [[ARRAY_DESTROY_DONE2]]:
diff --git a/clang/test/CodeGenCXX/cxx0x-initializer-references.cpp b/clang/test/CodeGenCXX/cxx0x-initializer-references.cpp
index b19ca1d861b11c7e1e5283d56614386c0c7ed3d3..419525d0544d52e736e8e33ba32516d9088f9aaf 100644
--- a/clang/test/CodeGenCXX/cxx0x-initializer-references.cpp
+++ b/clang/test/CodeGenCXX/cxx0x-initializer-references.cpp
@@ -52,11 +52,10 @@ namespace reference {
     // CHECK-NEXT: store ptr %{{.*}}, ptr %{{.*}}, align
     const A &ra1{1, i};
 
-    // CHECK-NEXT: getelementptr inbounds [3 x i32], ptr %{{.*}}, i{{32|64}} 0, i{{32|64}} 0
     // CHECK-NEXT: store i32 1
     // CHECK-NEXT: getelementptr inbounds i32, ptr %{{.*}}, i{{32|64}} 1
     // CHECK-NEXT: store i32 2
-    // CHECK-NEXT: getelementptr inbounds i32, ptr %{{.*}}, i{{32|64}} 1
+    // CHECK-NEXT: getelementptr inbounds i32, ptr %{{.*}}, i{{32|64}} 2
     // CHECK-NEXT: %[[I2:.*]] = load i32, ptr
     // CHECK-NEXT: store i32 %[[I2]]
     // CHECK-NEXT: store ptr %{{.*}}, ptr %{{.*}}, align
diff --git a/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist-startend.cpp b/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist-startend.cpp
index d9e4c5d5403c25143f8b454e84a2afc6903f8849..e2d566121331fdc7b57354669c2f4dfb48ae91c3 100644
--- a/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist-startend.cpp
+++ b/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist-startend.cpp
@@ -40,8 +40,7 @@ void fn1(int i) {
   // CHECK-LABEL: define{{.*}} void @_Z3fn1i
   // temporary array
   // CHECK: [[array:%[^ ]+]] = alloca [3 x i32]
-  // CHECK: getelementptr inbounds [3 x i32], ptr [[array]], i{{32|64}} 0
-  // CHECK-NEXT: store i32 1, ptr
+  // CHECK:      store i32 1, ptr
   // CHECK-NEXT: getelementptr
   // CHECK-NEXT: store
   // CHECK-NEXT: getelementptr
diff --git a/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist.cpp b/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist.cpp
index aa2f078a5fb0ca61e602a2cb8d7cbc3ecd889e22..3d0cf968bfd54cbaadef1a18fd66e732e99cea99 100644
--- a/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist.cpp
+++ b/clang/test/CodeGenCXX/cxx0x-initializer-stdinitializerlist.cpp
@@ -122,8 +122,7 @@ void fn1(int i) {
   // X86: [[array:%[^ ]+]] = alloca [3 x i32]
   // AMDGCN: [[alloca:%[^ ]+]] = alloca [3 x i32], align 4, addrspace(5)
   // AMDGCN: [[array:%[^ ]+]] ={{.*}} addrspacecast ptr addrspace(5) [[alloca]] to ptr
-  // CHECK: getelementptr inbounds [3 x i32], ptr [[array]], i{{32|64}} 0
-  // CHECK-NEXT: store i32 1, ptr
+  // CHECK:      store i32 1, ptr
   // CHECK-NEXT: getelementptr
   // CHECK-NEXT: store
   // CHECK-NEXT: getelementptr
diff --git a/clang/test/CodeGenCXX/cxx11-initializer-array-new.cpp b/clang/test/CodeGenCXX/cxx11-initializer-array-new.cpp
index 48ee019a55f555aff3da0890eee613ec5e26700d..d86c712372cd89cc2ead093b586e36b1d086eec1 100644
--- a/clang/test/CodeGenCXX/cxx11-initializer-array-new.cpp
+++ b/clang/test/CodeGenCXX/cxx11-initializer-array-new.cpp
@@ -16,22 +16,20 @@ void *p = new S[2][3]{ { 1, 2, 3 }, { 4, 5, 6 } };
 // { 1, 2, 3 }
 //
 //
-// CHECK: %[[S_0_0:.*]] = getelementptr inbounds [3 x %[[S:.*]]], ptr %[[START_AS_i8]], i64 0, i64 0
-// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_0]], i32 noundef 1)
-// CHECK: %[[S_0_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_0_0]], i64 1
+// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[START_AS_i8]], i32 noundef 1)
+// CHECK: %[[S_0_1:.*]] = getelementptr inbounds %[[S:.+]], ptr %[[START_AS_i8]], i64 1
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_1]], i32 noundef 2)
-// CHECK: %[[S_0_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_0_1]], i64 1
+// CHECK: %[[S_0_2:.*]] = getelementptr inbounds %[[S]], ptr %[[START_AS_i8]], i64 2
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_2]], i32 noundef 3)
 //
 // { 4, 5, 6 }
 //
 // CHECK: %[[S_1:.*]] = getelementptr inbounds [3 x %[[S]]], ptr %[[START_AS_i8]], i64 1
 //
-// CHECK: %[[S_1_0:.*]] = getelementptr inbounds [3 x %[[S]]], ptr %[[S_1]], i64 0, i64 0
-// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_0]], i32 noundef 4)
-// CHECK: %[[S_1_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1_0]], i64 1
+// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1]], i32 noundef 4)
+// CHECK: %[[S_1_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1]], i64 1
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_1]], i32 noundef 5)
-// CHECK: %[[S_1_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1_1]], i64 1
+// CHECK: %[[S_1_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1]], i64 2
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_2]], i32 noundef 6)
 //
 // CHECK-NOT: br i1
@@ -57,22 +55,20 @@ void *q = new S[n][3]{ { 1, 2, 3 }, { 4, 5, 6 } };
 // { 1, 2, 3 }
 //
 //
-// CHECK: %[[S_0_0:.*]] = getelementptr inbounds [3 x %[[S]]], ptr %[[START_AS_i8]], i64 0, i64 0
-// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_0]], i32 noundef 1)
-// CHECK: %[[S_0_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_0_0]], i64 1
+// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[START_AS_i8]], i32 noundef 1)
+// CHECK: %[[S_0_1:.*]] = getelementptr inbounds %[[S]], ptr %[[START_AS_i8]], i64 1
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_1]], i32 noundef 2)
-// CHECK: %[[S_0_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_0_1]], i64 1
+// CHECK: %[[S_0_2:.*]] = getelementptr inbounds %[[S]], ptr %[[START_AS_i8]], i64 2
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_0_2]], i32 noundef 3)
 //
 // { 4, 5, 6 }
 //
 // CHECK: %[[S_1:.*]] = getelementptr inbounds [3 x %[[S]]], ptr %[[START_AS_i8]], i64 1
 //
-// CHECK: %[[S_1_0:.*]] = getelementptr inbounds [3 x %[[S]]], ptr %[[S_1]], i64 0, i64 0
-// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_0]], i32 noundef 4)
-// CHECK: %[[S_1_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1_0]], i64 1
+// CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1]], i32 noundef 4)
+// CHECK: %[[S_1_1:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1]], i64 1
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_1]], i32 noundef 5)
-// CHECK: %[[S_1_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1_1]], i64 1
+// CHECK: %[[S_1_2:.*]] = getelementptr inbounds %[[S]], ptr %[[S_1]], i64 2
 // CHECK: call void @_ZN1SC1Ei(ptr {{[^,]*}} %[[S_1_2]], i32 noundef 6)
 //
 // And the rest.
@@ -114,13 +110,12 @@ void *r = new T[n][3]{ { 1, 2, 3 }, { 4, 5, 6 } };
 // { 1, 2, 3 }
 //
 //
-// CHECK: %[[T_0_0:.*]] = getelementptr inbounds [3 x %[[T:.*]]], ptr %[[ALLOC]], i64 0, i64 0
-// CHECK: %[[T_0_0_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_0_0]], i32 0, i32 0
+// CHECK: %[[T_0_0_0:.*]] = getelementptr inbounds %[[T:.+]], ptr %[[ALLOC]], i32 0, i32 0
 // CHECK: store i32 1, ptr %[[T_0_0_0]]
-// CHECK: %[[T_0_1:.*]] = getelementptr inbounds %[[T]], ptr %[[T_0_0]], i64 1
+// CHECK: %[[T_0_1:.*]] = getelementptr inbounds %[[T]], ptr %[[ALLOC]], i64 1
 // CHECK: %[[T_0_1_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_0_1]], i32 0, i32 0
 // CHECK: store i32 2, ptr %[[T_0_1_0]]
-// CHECK: %[[T_0_2:.*]] = getelementptr inbounds %[[T]], ptr %[[T_0_1]], i64 1
+// CHECK: %[[T_0_2:.*]] = getelementptr inbounds %[[T]], ptr %[[ALLOC]], i64 2
 // CHECK: %[[T_0_2_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_0_2]], i32 0, i32 0
 // CHECK: store i32 3, ptr %[[T_0_2_0]]
 //
@@ -128,13 +123,12 @@ void *r = new T[n][3]{ { 1, 2, 3 }, { 4, 5, 6 } };
 //
 // CHECK: %[[T_1:.*]] = getelementptr inbounds [3 x %[[T]]], ptr %[[ALLOC]], i64 1
 //
-// CHECK: %[[T_1_0:.*]] = getelementptr inbounds [3 x %[[T]]], ptr %[[T_1]], i64 0, i64 0
-// CHECK: %[[T_1_0_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1_0]], i32 0, i32 0
+// CHECK: %[[T_1_0_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1]], i32 0, i32 0
 // CHECK: store i32 4, ptr %[[T_1_0_0]]
-// CHECK: %[[T_1_1:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1_0]], i64 1
+// CHECK: %[[T_1_1:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1]], i64 1
 // CHECK: %[[T_1_1_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1_1]], i32 0, i32 0
 // CHECK: store i32 5, ptr %[[T_1_1_0]]
-// CHECK: %[[T_1_2:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1_1]], i64 1
+// CHECK: %[[T_1_2:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1]], i64 2
 // CHECK: %[[T_1_2_0:.*]] = getelementptr inbounds %[[T]], ptr %[[T_1_2]], i32 0, i32 0
 // CHECK: store i32 6, ptr %[[T_1_2_0]]
 //
diff --git a/clang/test/CodeGenCXX/debug-info-struct-align.cpp b/clang/test/CodeGenCXX/debug-info-struct-align.cpp
index 1269cbce83ef0a03894eb0ee83450f6d6f8f8c4e..cd91f4c302ddc6a191639c88725899abfdc4164b 100644
--- a/clang/test/CodeGenCXX/debug-info-struct-align.cpp
+++ b/clang/test/CodeGenCXX/debug-info-struct-align.cpp
@@ -25,3 +25,11 @@ struct MyType2 {
 MyType2 mt2;
 
 static_assert(alignof(MyType2) == 1, "alignof MyType2 is wrong");
+
+#pragma pack(1)
+struct MyType3 {
+  int m;
+};
+MyType3 mt3;
+
+static_assert(alignof(MyType3) == 1, "alignof MyType3 is wrong");
diff --git a/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a0673b96626d15e9173d0490c5ff1f55d584c1a6
--- /dev/null
+++ b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp
@@ -0,0 +1,181 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature
+// REQUIRES: webassembly-registered-target
+
+// Simple calls to known variadic functions that are completely elided when
+// optimisations are on This is a functional check that the expand-variadic pass
+// is consistent with clang's va_arg handling
+
+// When expand-variadics is added to the default pipeline, clang -O1 will
+// suffice here -Wno-varargs avoids warning second argument to 'va_start' is not
+// the last named parameter
+
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -Wno-varargs -O1 -emit-llvm -o - | opt - -S --passes='module(expand-variadics,default<O1>)' --expand-variadics-override=optimize -o - | FileCheck %s
+
+#include <stdarg.h>
+#include <stdint.h>
+
+template <typename X, typename Y> static X first(...) {
+  va_list va;
+  __builtin_va_start(va, 0);
+  X r = va_arg(va, X);
+  va_end(va);
+  return r;
+}
+
+template <typename X, typename Y> static Y second(...) {
+  va_list va;
+  __builtin_va_start(va, 0);
+  va_arg(va, X);
+  Y r = va_arg(va, Y);
+  va_end(va);
+  return r;
+}
+
+extern "C" {
+
+// CHECK-LABEL: define {{[^@]+}}@first_pair_i32
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], i32 noundef [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_pair_i32(int x, int y) { return first<int, int>(x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_pair_i32
+// CHECK-SAME: (i32 noundef [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_pair_i32(int x, int y) { return second<int, int>(x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@first_pair_f64
+// CHECK-SAME: (double noundef returned [[X:%.*]], double noundef [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double [[X]]
+//
+double first_pair_f64(double x, double y) {
+  return first<double, double>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@second_pair_f64
+// CHECK-SAME: (double noundef [[X:%.*]], double noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double [[Y]]
+//
+double second_pair_f64(double x, double y) {
+  return second<double, double>(x, y);
+}
+}
+
+extern "C" {
+
+// CHECK-LABEL: define {{[^@]+}}@first_i32_f64
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], double noundef [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_i32_f64(int x, double y) { return first<int, double>(x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_i32_f64
+// CHECK-SAME: (i32 noundef [[X:%.*]], double noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double [[Y]]
+//
+double second_i32_f64(int x, double y) { return second<int, double>(x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@first_f64_i32
+// CHECK-SAME: (double noundef returned [[X:%.*]], i32 noundef [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double [[X]]
+//
+double first_f64_i32(double x, int y) { return first<double, int>(x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_f64_i32
+// CHECK-SAME: (double noundef [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_f64_i32(double x, int y) { return second<double, int>(x, y); }
+}
+
+extern "C" {
+typedef uint64_t ulong2 __attribute__((__vector_size__(16), __aligned__(16)));
+
+// CHECK-LABEL: define {{[^@]+}}@first_i32_ulong2
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_i32_ulong2(int x, ulong2 *y) { return first<int, ulong2>(x, *y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_i32_ulong2
+// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void second_i32_ulong2(int x, ulong2 *y, ulong2 *r) {
+  *r = second<int, ulong2>(x, *y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@first_ulong2_i32
+// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X]], align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void first_ulong2_i32(ulong2 *x, int y, ulong2 *r) {
+  *r = first<ulong2, int>(*x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@second_ulong2_i32
+// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_ulong2_i32(ulong2 *x, int y) { return second<ulong2, int>(*x, y); }
+}
+
+// ascending alignment
+typedef struct {
+  char c;
+  short s;
+  int i;
+  long l;
+  float f;
+  double d;
+} asc;
+
+extern "C" {
+
+// CHECK-LABEL: define {{[^@]+}}@first_i32_asc
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_i32_asc(int x, asc *y) { return first<int, asc>(x, *y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_i32_asc
+// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[Y]], i32 24, i1 false)
+// CHECK-NEXT:    ret void
+//
+void second_i32_asc(int x, asc *y, asc *r) { *r = second<int, asc>(x, *y); }
+
+// CHECK-LABEL: define {{[^@]+}}@first_asc_i32
+// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[X]], i32 24, i1 false)
+// CHECK-NEXT:    ret void
+//
+void first_asc_i32(asc *x, int y, asc *r) { *r = first<asc, int>(*x, y); }
+
+// CHECK-LABEL: define {{[^@]+}}@second_asc_i32
+// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_asc_i32(asc *x, int y) { return second<asc, int>(*x, y); }
+}
diff --git a/clang/test/CodeGenCXX/partial-destruction.cpp b/clang/test/CodeGenCXX/partial-destruction.cpp
index 840e43880293b9fa0a7478b115b3ceb9376042c5..8ceb4b9bbedeaaea29ee9f654e22d18eef0281f0 100644
--- a/clang/test/CodeGenCXX/partial-destruction.cpp
+++ b/clang/test/CodeGenCXX/partial-destruction.cpp
@@ -20,15 +20,14 @@ namespace test0 {
   // CHECK-NEXT: [[SEL:%.*]] = alloca i32
 
   // Initialize.
-  // CHECK-NEXT: [[E_BEGIN:%.*]] = getelementptr inbounds [10 x [[A]]], ptr [[AS]], i64 0, i64 0
-  // CHECK-NEXT: store ptr [[E_BEGIN]], ptr [[ENDVAR]]
-  // CHECK-NEXT: invoke void @_ZN5test01AC1Ei(ptr {{[^,]*}} [[E_BEGIN]], i32 noundef 5)
-  // CHECK:      [[E1:%.*]] = getelementptr inbounds [[A]], ptr [[E_BEGIN]], i64 1
+  // CHECK-NEXT: store ptr [[AS]], ptr [[ENDVAR]]
+  // CHECK-NEXT: invoke void @_ZN5test01AC1Ei(ptr {{[^,]*}} [[AS]], i32 noundef 5)
+  // CHECK:      [[E1:%.*]] = getelementptr inbounds [[A]], ptr [[AS]], i64 1
   // CHECK-NEXT: store ptr [[E1]], ptr [[ENDVAR]]
   // CHECK-NEXT: invoke void @_ZN5test01AC1Ei(ptr {{[^,]*}} [[E1]], i32 noundef 7)
-  // CHECK:      [[E2:%.*]] = getelementptr inbounds [[A]], ptr [[E1]], i64 1
+  // CHECK:      [[E2:%.*]] = getelementptr inbounds [[A]], ptr [[AS]], i64 2
   // CHECK-NEXT: store ptr [[E2]], ptr [[ENDVAR]]
-  // CHECK-NEXT: [[E_END:%.*]] = getelementptr inbounds [[A]], ptr [[E_BEGIN]], i64 10
+  // CHECK-NEXT: [[E_END:%.*]] = getelementptr inbounds [[A]], ptr [[AS]], i64 10
   // CHECK-NEXT: br label
   // CHECK:      [[E_CUR:%.*]] = phi ptr [ [[E2]], {{%.*}} ], [ [[E_NEXT:%.*]], {{%.*}} ]
   // CHECK-NEXT: invoke void @_ZN5test01AC1Ev(ptr {{[^,]*}} [[E_CUR]])
@@ -56,13 +55,13 @@ namespace test0 {
   // CHECK:      landingpad { ptr, i32 }
   // CHECK-NEXT:   cleanup
   // CHECK:      [[PARTIAL_END:%.*]] = load ptr, ptr [[ENDVAR]]
-  // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[E_BEGIN]], [[PARTIAL_END]]
+  // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[AS]], [[PARTIAL_END]]
   // CHECK-NEXT: br i1 [[T0]],
   // CHECK:      [[E_AFTER:%.*]] = phi ptr [ [[PARTIAL_END]], {{%.*}} ], [ [[E_CUR:%.*]], {{%.*}} ]
   // CHECK-NEXT: [[E_CUR]] = getelementptr inbounds [[A]], ptr [[E_AFTER]], i64 -1
   // CHECKv03-NEXT: invoke void @_ZN5test01AD1Ev(ptr {{[^,]*}} [[E_CUR]])
   // CHECKv11-NEXT: call   void @_ZN5test01AD1Ev(ptr {{[^,]*}} [[E_CUR]])
-  // CHECK:      [[T0:%.*]] = icmp eq ptr [[E_CUR]], [[E_BEGIN]]
+  // CHECK:      [[T0:%.*]] = icmp eq ptr [[E_CUR]], [[AS]]
   // CHECK-NEXT: br i1 [[T0]],
 
   // Primary EH destructor.
@@ -189,25 +188,22 @@ namespace test4 {
 }
 // CHECK-LABEL: define{{.*}} void @_ZN5test44testEv()
 // CHECK:       [[ARRAY:%.*]] = alloca [2 x [3 x [[A:%.*]]]], align
-// CHECK:       [[A0:%.*]] = getelementptr inbounds [2 x [3 x [[A]]]], ptr [[ARRAY]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A0]],
-// CHECK-NEXT:  [[A00:%.*]] = getelementptr inbounds [3 x [[A]]], ptr [[A0]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A00]],
-// CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A00]], i32 noundef 0)
-// CHECK:       [[A01:%.*]] = getelementptr inbounds [[A]], ptr [[A00]], i64 1
+// CHECK:       store ptr [[ARRAY]],
+// CHECK-NEXT:  store ptr [[ARRAY]],
+// CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[ARRAY]], i32 noundef 0)
+// CHECK:       [[A01:%.*]] = getelementptr inbounds [[A]], ptr [[ARRAY]], i64 1
 // CHECK-NEXT:  store ptr [[A01]],
 // CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A01]], i32 noundef 1)
-// CHECK:       [[A02:%.*]] = getelementptr inbounds [[A]], ptr [[A01]], i64 1
+// CHECK:       [[A02:%.*]] = getelementptr inbounds [[A]], ptr [[ARRAY]], i64 2
 // CHECK-NEXT:  store ptr [[A02]],
 // CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A02]], i32 noundef 2)
-// CHECK:       [[A1:%.*]] = getelementptr inbounds [3 x [[A]]], ptr [[A0]], i64 1
+// CHECK:       [[A1:%.*]] = getelementptr inbounds [3 x [[A]]], ptr [[ARRAY]], i64 1
 // CHECK-NEXT:  store ptr [[A1]],
-// CHECK-NEXT:  [[A10:%.*]] = getelementptr inbounds [3 x [[A]]], ptr [[A1]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A10]],
-// CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A10]], i32 noundef 3)
-// CHECK:       [[A11:%.*]] = getelementptr inbounds [[A]], ptr [[A10]], i64 1
+// CHECK-NEXT:  store ptr [[A1]],
+// CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A1]], i32 noundef 3)
+// CHECK:       [[A11:%.*]] = getelementptr inbounds [[A]], ptr [[A1]], i64 1
 // CHECK-NEXT:  store ptr [[A11]],
 // CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A11]], i32 noundef 4)
-// CHECK:       [[A12:%.*]] = getelementptr inbounds [[A]], ptr [[A11]], i64 1
+// CHECK:       [[A12:%.*]] = getelementptr inbounds [[A]], ptr [[A1]], i64 2
 // CHECK-NEXT:  store ptr [[A12]],
 // CHECK-NEXT:  invoke void @_ZN5test41AC1Ej(ptr {{[^,]*}} [[A12]], i32 noundef 5)
diff --git a/clang/test/CodeGenCXX/pointers-to-data-members.cpp b/clang/test/CodeGenCXX/pointers-to-data-members.cpp
index 29f1c3f48e3ac04308fa8ef13d6c6cd483f84257..cf1d6c018409d2a64470d9f118d225505594e743 100644
--- a/clang/test/CodeGenCXX/pointers-to-data-members.cpp
+++ b/clang/test/CodeGenCXX/pointers-to-data-members.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 %s -emit-llvm -o %t.ll -triple=x86_64-apple-darwin10
+// RUN: %clang_cc1 %s -emit-llvm -o %t.ll -triple=x86_64-apple-darwin10 -fexperimental-new-constant-interpreter
 // RUN: FileCheck %s < %t.ll
 // RUN: FileCheck -check-prefix=CHECK-GLOBAL %s < %t.ll
 
diff --git a/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2487e0fcd4343a2aac8ebfa547a6104cb0dd43ca
--- /dev/null
+++ b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp
@@ -0,0 +1,38 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: define spir_func void @_Z1fv(
+// CHECK-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X:%.*]] = alloca half, align 2
+// CHECK-NEXT:    [[Y:%.*]] = alloca half, align 2
+// CHECK-NEXT:    [[Z:%.*]] = alloca half, align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr [[X]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    [[ADD:%.*]] = fadd half [[TMP0]], [[TMP1]]
+// CHECK-NEXT:    store half [[ADD]], ptr [[Z]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = load half, ptr [[X]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    [[SUB:%.*]] = fsub half [[TMP2]], [[TMP3]]
+// CHECK-NEXT:    store half [[SUB]], ptr [[Z]], align 2
+// CHECK-NEXT:    [[TMP4:%.*]] = load half, ptr [[X]], align 2
+// CHECK-NEXT:    [[TMP5:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    [[MUL:%.*]] = fmul half [[TMP4]], [[TMP5]]
+// CHECK-NEXT:    store half [[MUL]], ptr [[Z]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load half, ptr [[X]], align 2
+// CHECK-NEXT:    [[TMP7:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv half [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    store half [[DIV]], ptr [[Z]], align 2
+// CHECK-NEXT:    ret void
+//
+void f() {
+  _Float16 x, y, z;
+
+  z = x + y;
+
+  z = x - y;
+
+  z = x * y;
+
+  z = x / y;
+}
diff --git a/clang/test/CodeGenCXX/template-param-objects-linkage.cpp b/clang/test/CodeGenCXX/template-param-objects-linkage.cpp
index 63e7d8c6468695d981742db0ba3f26674097dee5..9c148ed83753d86de958fbebde2a9060efa537f6 100644
--- a/clang/test/CodeGenCXX/template-param-objects-linkage.cpp
+++ b/clang/test/CodeGenCXX/template-param-objects-linkage.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 %s -emit-llvm -o - -fexperimental-new-constant-interpreter | FileCheck %s
 
 struct S { char buf[32]; };
 template<S s> constexpr const char* f() { return s.buf; }
diff --git a/clang/test/CodeGenCXX/temporaries.cpp b/clang/test/CodeGenCXX/temporaries.cpp
index 9f29d3b7327839515c3aebd40127cab921434482..186f4934873fa94814c6686a729d000b3e602758 100644
--- a/clang/test/CodeGenCXX/temporaries.cpp
+++ b/clang/test/CodeGenCXX/temporaries.cpp
@@ -477,9 +477,8 @@ namespace Elision {
     // CHECK-NEXT: call void @_ZN7Elision1AC1Ev(ptr {{[^,]*}} [[X]])
     A x;
 
-    // CHECK-NEXT: [[XS0:%.*]] = getelementptr inbounds [2 x [[A]]], ptr [[XS]], i64 0, i64 0
-    // CHECK-NEXT: call void @_ZN7Elision1AC1Ev(ptr {{[^,]*}} [[XS0]])
-    // CHECK-NEXT: [[XS1:%.*]] = getelementptr inbounds [[A]], ptr [[XS0]], i64 1
+    // CHECK-NEXT: call void @_ZN7Elision1AC1Ev(ptr {{[^,]*}} [[XS]])
+    // CHECK-NEXT: [[XS1:%.*]] = getelementptr inbounds [[A]], ptr [[XS]], i64 1
     // CHECK-NEXT: call void @_ZN7Elision1AC1ERKS0_(ptr {{[^,]*}} [[XS1]], ptr noundef {{(nonnull )?}}align {{[0-9]+}} dereferenceable({{[0-9]+}}) [[X]])
     A xs[] = { A(), x };
 
diff --git a/clang/test/CodeGenCXX/value-init.cpp b/clang/test/CodeGenCXX/value-init.cpp
index 42181be92ace9e1c80f285e5955e91a0b4ca3692..9e72769d87f696a894f4690f3f34db81dd339311 100644
--- a/clang/test/CodeGenCXX/value-init.cpp
+++ b/clang/test/CodeGenCXX/value-init.cpp
@@ -205,11 +205,9 @@ namespace test6 {
   // CHECK-LABEL:    define{{.*}} void @_ZN5test64testEv()
   // CHECK:      [[ARR:%.*]] = alloca [10 x [20 x [[A:%.*]]]],
 
-  // CHECK-NEXT: [[INNER:%.*]] = getelementptr inbounds [10 x [20 x [[A]]]], ptr [[ARR]], i64 0, i64 0
-  // CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[INNER]], i64 0, i64 0
-  // CHECK-NEXT: call void @_ZN5test61AC1Ei(ptr {{[^,]*}} [[T0]], i32 noundef 5)
-  // CHECK-NEXT: [[BEGIN:%.*]] = getelementptr inbounds [[A]], ptr [[T0]], i64 1
-  // CHECK-NEXT: [[END:%.*]] = getelementptr inbounds [[A]], ptr [[T0]], i64 20
+  // CHECK-NEXT: call void @_ZN5test61AC1Ei(ptr {{[^,]*}} [[ARR]], i32 noundef 5)
+  // CHECK-NEXT: [[BEGIN:%.*]] = getelementptr inbounds [[A]], ptr [[ARR]], i64 1
+  // CHECK-NEXT: [[END:%.*]] = getelementptr inbounds [[A]], ptr [[ARR]], i64 20
   // CHECK-NEXT: br label
   // CHECK:      [[CUR:%.*]] = phi ptr [ [[BEGIN]], {{%.*}} ], [ [[NEXT:%.*]], {{%.*}} ]
   // CHECK-NEXT: call void @_ZN5test61AC1Ev(ptr {{[^,]*}} [[CUR]])
@@ -217,13 +215,15 @@ namespace test6 {
   // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[NEXT]], [[END]]
   // CHECK-NEXT: br i1
 
-  // CHECK:      [[BEGIN:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[INNER]], i64 1
-  // CHECK-NEXT: [[END:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[INNER]], i64 10
+  // CHECK:      [[BEGIN:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[ARR]], i64 1
+  // CHECK-NEXT: [[END:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[ARR]], i64 10
   // CHECK-NEXT: br label
-  // CHECK:      [[CUR:%.*]] = phi ptr [ [[BEGIN]], {{%.*}} ], [ [[NEXT:%.*]], {{%.*}} ]
-
   // Inner loop.
-  // CHECK-NEXT: [[IBEGIN:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[CUR]], i{{32|64}} 0, i{{32|64}} 0
+  // CHECK-CXX98: [[CUR:%.*]] = phi ptr [ [[BEGIN]], {{%.*}} ], [ [[NEXT:%.*]], {{%.*}} ]
+
+  // CHECK-CXX98: [[IBEGIN:%.*]] = getelementptr inbounds [20 x [[A]]], ptr [[CUR]], i{{32|64}} 0, i{{32|64}} 0
+  // CHECK-CXX17: [[IBEGIN:%.*]] = phi ptr [ [[BEGIN]], {{%.*}} ], [ [[NEXT:%.*]], {{%.*}} ]
+
   // CHECK-NEXT: [[IEND:%.*]] = getelementptr inbounds [[A]], ptr [[IBEGIN]], i64 20
   // CHECK-NEXT: br label
   // CHECK:      [[ICUR:%.*]] = phi ptr [ [[IBEGIN]], {{%.*}} ], [ [[INEXT:%.*]], {{%.*}} ]
@@ -232,7 +232,8 @@ namespace test6 {
   // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[INEXT]], [[IEND]]
   // CHECK-NEXT: br i1 [[T0]],
 
-  // CHECK:      [[NEXT]] = getelementptr inbounds [20 x [[A]]], ptr [[CUR]], i64 1
+  // CHECK-CXX98: [[NEXT]] = getelementptr inbounds [20 x [[A]]], ptr [[CUR]], i64 1
+  // CHECK-CXX17: [[NEXT]] = getelementptr inbounds [20 x [[A]]], ptr [[IBEGIN]], i64 1
   // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[NEXT]], [[END]]
   // CHECK-NEXT: br i1 [[T0]]
   // CHECK:      ret void
diff --git a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp b/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp
index 06cc2069dbe9aed686fbb8e6295ca38ae8990bb8..d71c2c558996a7adb7ab107228a51a24767a9ad1 100644
--- a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp
+++ b/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp
@@ -43,14 +43,11 @@ coroutine ArrayInitCoro() {
   // CHECK: %cleanup.isactive = alloca i1, align 1
   Printy arr[2] = {
     Printy("a"),
-    // CHECK:       %arrayinit.begin = getelementptr inbounds [2 x %struct.Printy], ptr %arr.reload.addr, i64 0, i64 0
-    // CHECK-NEXT:  %arrayinit.begin.spill.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 10
-    // CHECK-NEXT:  store ptr %arrayinit.begin, ptr %arrayinit.begin.spill.addr, align 8
-    // CHECK-NEXT:  store i1 true, ptr %cleanup.isactive.reload.addr, align 1
-    // CHECK-NEXT:  store ptr %arrayinit.begin, ptr %arrayinit.endOfInit.reload.addr, align 8
-    // CHECK-NEXT:  call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arrayinit.begin, ptr noundef @.str)
-    // CHECK-NEXT:  %arrayinit.element = getelementptr inbounds %struct.Printy, ptr %arrayinit.begin, i64 1
-    // CHECK-NEXT:  %arrayinit.element.spill.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 11
+    // CHECK:       store i1 true, ptr %cleanup.isactive.reload.addr, align 1
+    // CHECK-NEXT:  store ptr %arr.reload.addr, ptr %arrayinit.endOfInit.reload.addr, align 8
+    // CHECK-NEXT:  call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arr.reload.addr, ptr noundef @.str)
+    // CHECK-NEXT:  %arrayinit.element = getelementptr inbounds %struct.Printy, ptr %arr.reload.addr, i64 1
+    // CHECK-NEXT:  %arrayinit.element.spill.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 10
     // CHECK-NEXT:  store ptr %arrayinit.element, ptr %arrayinit.element.spill.addr, align 8
     // CHECK-NEXT:  store ptr %arrayinit.element, ptr %arrayinit.endOfInit.reload.addr, align 8
     co_await Awaiter{}
@@ -64,7 +61,7 @@ coroutine ArrayInitCoro() {
   // CHECK:         br label %cleanup{{.*}}
 
   // CHECK:       await.ready:
-  // CHECK-NEXT:    %arrayinit.element.reload.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 11
+  // CHECK-NEXT:    %arrayinit.element.reload.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 10
   // CHECK-NEXT:    %arrayinit.element.reload = load ptr, ptr %arrayinit.element.reload.addr, align 8
   // CHECK-NEXT:    call void @_ZN7Awaiter12await_resumeEv
   // CHECK-NEXT:    store i1 false, ptr %cleanup.isactive.reload.addr, align 1
@@ -75,7 +72,7 @@ coroutine ArrayInitCoro() {
   // CHECK-NEXT:    br i1 %cleanup.is_active, label %cleanup.action, label %cleanup.done
 
   // CHECK:       cleanup.action:
-  // CHECK:         %arraydestroy.isempty = icmp eq ptr %arrayinit.begin.reload{{.*}}, %{{.*}}
+  // CHECK:         %arraydestroy.isempty = icmp eq ptr %arr.reload.addr, %{{.*}}
   // CHECK-NEXT:    br i1 %arraydestroy.isempty, label %arraydestroy.done{{.*}}, label %arraydestroy.body.from.cleanup.action
   // Ignore rest of the array cleanup.
 }
diff --git a/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp b/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp
index 9fcdc460482e7e26f0c0d265054f91b96586d122..c575f49ff697165513756f217f867f49687b0888 100644
--- a/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp
+++ b/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -triple spirv64 -x hip -emit-llvm -fcuda-is-device \
 // RUN:   -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -emit-llvm -fcuda-is-device \
+// RUN:   -o - %s | FileCheck %s
 
 #define __device__ __attribute__((device))
 #define __shared__ __attribute__((shared))
diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp b/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8226a109d8b8d9bc035ea25e33e812bf9c06961b
--- /dev/null
+++ b/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp
@@ -0,0 +1,27 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -aux-triple x86_64-pc-windows-msvc -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+
+// Unlike OpenCL, HIP depends on the C++ interpration of "unsigned long", which
+// is 64 bits long on Linux and 32 bits long on Windows. The return type of the
+// ballot intrinsic needs to be a 64 bit integer on both platforms. This test
+// cross-compiles to Windows to confirm that the return type is indeed 64 bits
+// on Windows.
+
+#define __device__ __attribute__((device))
+
+// CHECK-LABEL: define spir_func noundef i64 @_Z3fooi(
+// CHECK-SAME: i32 noundef [[P:%.*]]) addrspace(4) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[P_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
+// CHECK-NEXT:    [[P_ADDR_ASCAST:%.*]] = addrspacecast ptr [[P_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    store i32 [[P]], ptr addrspace(4) [[P_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[P_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT:    [[TMP1:%.*]] = call addrspace(4) i64 @llvm.amdgcn.ballot.i64(i1 [[TOBOOL]])
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+__device__ unsigned long long foo(int p) {
+  return __builtin_amdgcn_ballot_w64(p);
+}
diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip b/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip
new file mode 100644
index 0000000000000000000000000000000000000000..2b785200e8eeab97a92589886857f3cc6818ad73
--- /dev/null
+++ b/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -fcuda-is-device -emit-llvm %s \
+// RUN:   -o - | FileCheck %s
+
+constexpr static int OpCtrl()
+{
+    return 15 + 1;
+}
+
+constexpr static int RowMask()
+{
+    return 3 + 1;
+}
+
+constexpr static int BankMask()
+{
+    return 2 + 1;
+}
+
+constexpr static bool BountCtrl()
+{
+    return true & false;
+}
+
+// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 16, i32 0, i32 0, i1 false)
+__attribute__((global)) void test_update_dpp_const_fold_imm_operand_2(int* out, int a, int b)
+{
+  *out = __builtin_amdgcn_update_dpp(a, b, OpCtrl(), 0, 0, false);
+}
+
+// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 4, i32 0, i1 false)
+__attribute__((global)) void test_update_dpp_const_fold_imm_operand_3(int* out, int a, int b)
+{
+  *out = __builtin_amdgcn_update_dpp(a, b, 0, RowMask(), 0, false);
+}
+
+// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 0, i32 3, i1 false)
+__attribute__((global)) void test_update_dpp_const_fold_imm_operand_4(int* out, int a, int b)
+{
+  *out = __builtin_amdgcn_update_dpp(a, b, 0, 0, BankMask(), false);
+}
+
+// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 0, i32 0, i1 false)
+__attribute__((global)) void test_update_dpp_const_fold_imm_operand_5(int* out, int a, int b)
+{
+  *out = __builtin_amdgcn_update_dpp(a, b, 0, 0, 0, BountCtrl());
+}
diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-half.hip b/clang/test/CodeGenHIP/spirv-amdgcn-half.hip
new file mode 100644
index 0000000000000000000000000000000000000000..2caf766d943b11cbb6f0bfd45e74a8467ee3b8fd
--- /dev/null
+++ b/clang/test/CodeGenHIP/spirv-amdgcn-half.hip
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+
+#define __device__ __attribute__((device))
+
+// CHECK-LABEL: @_Z2d0DF16_
+// CHECK: fpext
+__device__ float d0(_Float16 x) {
+  return x;
+}
+
+// CHECK-LABEL: @_Z2d1f
+// CHECK: fptrunc
+__device__ _Float16 d1(float x) {
+  return x;
+}
diff --git a/clang/test/CodeGenObjC/arc-ternary-op.m b/clang/test/CodeGenObjC/arc-ternary-op.m
index 87167d911ea8ef1bec8d3022174ce35809b288cc..d633851ac76174e76e2e1431beb2efe6437cdeeb 100644
--- a/clang/test/CodeGenObjC/arc-ternary-op.m
+++ b/clang/test/CodeGenObjC/arc-ternary-op.m
@@ -153,22 +153,20 @@ void test3(int cond) {
   // CHECK: %[[_COMPOUNDLITERAL1:.*]] = alloca [2 x ptr], align 8
   // CHECK: %[[CLEANUP_COND4:.*]] = alloca i1, align 1
 
-  // CHECK: %[[ARRAYINIT_BEGIN:.*]] = getelementptr inbounds [2 x ptr], ptr %[[_COMPOUNDLITERAL]], i64 0, i64 0
   // CHECK: %[[V2:.*]] = load ptr, ptr @g0, align 8
   // CHECK: %[[V3:.*]] = call ptr @llvm.objc.retain(ptr %[[V2]])
-  // CHECK: store ptr %[[V3]], ptr %[[ARRAYINIT_BEGIN]], align 8
-  // CHECK: %[[ARRAYINIT_ELEMENT:.*]] = getelementptr inbounds ptr, ptr %[[ARRAYINIT_BEGIN]], i64 1
+  // CHECK: store ptr %[[V3]], ptr %[[_COMPOUNDLITERAL]], align 8
+  // CHECK: %[[ARRAYINIT_ELEMENT:.*]] = getelementptr inbounds ptr, ptr %[[_COMPOUNDLITERAL]], i64 1
   // CHECK: %[[V4:.*]] = load ptr, ptr @g1, align 8
   // CHECK: %[[V5:.*]] = call ptr @llvm.objc.retain(ptr %[[V4]])
   // CHECK: store ptr %[[V5]], ptr %[[ARRAYINIT_ELEMENT]], align 8
   // CHECK: store i1 true, ptr %[[CLEANUP_COND]], align 1
   // CHECK: %[[ARRAYDECAY:.*]] = getelementptr inbounds [2 x ptr], ptr %[[_COMPOUNDLITERAL]], i64 0, i64 0
 
-  // CHECK: %[[ARRAYINIT_BEGIN2:.*]] = getelementptr inbounds [2 x ptr], ptr %[[_COMPOUNDLITERAL1]], i64 0, i64 0
   // CHECK: %[[V6:.*]] = load ptr, ptr @g1, align 8
   // CHECK: %[[V7:.*]] = call ptr @llvm.objc.retain(ptr %[[V6]])
-  // CHECK: store ptr %[[V7]], ptr %[[ARRAYINIT_BEGIN2]], align 8
-  // CHECK: %[[ARRAYINIT_ELEMENT3:.*]] = getelementptr inbounds ptr, ptr %[[ARRAYINIT_BEGIN2]], i64 1
+  // CHECK: store ptr %[[V7]], ptr %[[_COMPOUNDLITERAL1]], align 8
+  // CHECK: %[[ARRAYINIT_ELEMENT3:.*]] = getelementptr inbounds ptr, ptr %[[_COMPOUNDLITERAL1]], i64 1
   // CHECK: %[[V8:.*]] = load ptr, ptr @g0, align 8
   // CHECK: %[[V9:.*]] = call ptr @llvm.objc.retain(ptr %[[V8]])
   // CHECK: store ptr %[[V9]], ptr %[[ARRAYINIT_ELEMENT3]], align 8
diff --git a/clang/test/CodeGenObjC/arc.m b/clang/test/CodeGenObjC/arc.m
index aeead58f8131d70402af5f48fb7c2a92326aa42f..48ca14af3b2abd91a6e9268dfb72e4f6b59c3ff8 100644
--- a/clang/test/CodeGenObjC/arc.m
+++ b/clang/test/CodeGenObjC/arc.m
@@ -1377,11 +1377,10 @@ void test71(void) {
 // CHECK: %[[T:.*]] = alloca [2 x ptr], align 16
 // CHECK: %[[V0:.*]] = call ptr @llvm.objc.retain(ptr %[[A]])
 // CHECK: %[[V1:.*]] = call ptr @llvm.objc.retain(ptr %[[B]]) #2
-// CHECK: %[[ARRAYINIT_BEGIN:.*]] = getelementptr inbounds [2 x ptr], ptr %[[T]], i64 0, i64 0
 // CHECK: %[[V3:.*]] = load ptr, ptr %[[A_ADDR]], align 8, !tbaa !7
 // CHECK: %[[V4:.*]] = call ptr @llvm.objc.retain(ptr %[[V3]]) #2
-// CHECK: store ptr %[[V4]], ptr %[[ARRAYINIT_BEGIN]], align 8, !tbaa !7
-// CHECK: %[[ARRAYINIT_ELEMENT:.*]] = getelementptr inbounds ptr, ptr %[[ARRAYINIT_BEGIN]], i64 1
+// CHECK: store ptr %[[V4]], ptr %[[T]], align 8, !tbaa !7
+// CHECK: %[[ARRAYINIT_ELEMENT:.*]] = getelementptr inbounds ptr, ptr %[[T]], i64 1
 // CHECK: %[[V5:.*]] = load ptr, ptr %[[B_ADDR]], align 8, !tbaa !7
 // CHECK: %[[V6:.*]] = call ptr @llvm.objc.retain(ptr %[[V5]]) #2
 // CHECK: store ptr %[[V6]], ptr %[[ARRAYINIT_ELEMENT]], align 8, !tbaa !7
diff --git a/clang/test/CodeGenObjCXX/arc-exceptions.mm b/clang/test/CodeGenObjCXX/arc-exceptions.mm
index 709afa32ac7d81f833ccefa27a18f4cfef8beebd..3efe566a6a8b51a24bce981145074f376313bd3c 100644
--- a/clang/test/CodeGenObjCXX/arc-exceptions.mm
+++ b/clang/test/CodeGenObjCXX/arc-exceptions.mm
@@ -115,23 +115,20 @@ void test5(void) {
 }
 // CHECK-LABEL: define{{.*}} void @_Z5test5v()
 // CHECK:       [[ARRAY:%.*]] = alloca [2 x [2 x ptr]], align
-// CHECK:       [[A0:%.*]] = getelementptr inbounds [2 x [2 x ptr]], ptr [[ARRAY]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A0]],
-// CHECK-NEXT:  [[A00:%.*]] = getelementptr inbounds [2 x ptr], ptr [[A0]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A00]],
+// CHECK:       store ptr [[ARRAY]],
+// CHECK-NEXT:  store ptr [[ARRAY]],
 // CHECK-NEXT:  [[T0:%.*]] = invoke noundef ptr @_Z12test5_helperj(i32 noundef 0)
-// CHECK:       store ptr [[T0]], ptr [[A00]], align
-// CHECK-NEXT:  [[A01:%.*]] = getelementptr inbounds ptr, ptr [[A00]], i64 1
+// CHECK:       store ptr [[T0]], ptr [[ARRAY]], align
+// CHECK-NEXT:  [[A01:%.*]] = getelementptr inbounds ptr, ptr [[ARRAY]], i64 1
 // CHECK-NEXT:  store ptr [[A01]],
 // CHECK-NEXT:  [[T0:%.*]] = invoke noundef ptr @_Z12test5_helperj(i32 noundef 1)
 // CHECK:       store ptr [[T0]], ptr [[A01]], align
-// CHECK-NEXT:  [[A1:%.*]] = getelementptr inbounds [2 x ptr], ptr [[A0]], i64 1
+// CHECK-NEXT:  [[A1:%.*]] = getelementptr inbounds [2 x ptr], ptr [[ARRAY]], i64 1
+// CHECK-NEXT:  store ptr [[A1]],
 // CHECK-NEXT:  store ptr [[A1]],
-// CHECK-NEXT:  [[A10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[A1]], i64 0, i64 0
-// CHECK-NEXT:  store ptr [[A10]],
 // CHECK-NEXT:  [[T0:%.*]] = invoke noundef ptr @_Z12test5_helperj(i32 noundef 2)
-// CHECK:       store ptr [[T0]], ptr [[A10]], align
-// CHECK-NEXT:  [[A11:%.*]] = getelementptr inbounds ptr, ptr [[A10]], i64 1
+// CHECK:       store ptr [[T0]], ptr [[A1]], align
+// CHECK-NEXT:  [[A11:%.*]] = getelementptr inbounds ptr, ptr [[A1]], i64 1
 // CHECK-NEXT:  store ptr [[A11]],
 // CHECK-NEXT:  [[T0:%.*]] = invoke noundef ptr @_Z12test5_helperj(i32 noundef 3)
 // CHECK:       store ptr [[T0]], ptr [[A11]], align
diff --git a/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl b/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl
index 619a9a99568e203730124a453ea92d5599b0939a..40a523f0aa0bfd49db9d72d5eb58c3100774d6aa 100644
--- a/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl
+++ b/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl
@@ -1,15 +1,16 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s
 
 // CHECK-LABEL: @use_flat_scratch_name
 kernel void use_flat_scratch_name()
 {
-// CHECK: tail call void asm sideeffect "s_mov_b64 flat_scratch, 0", "~{flat_scratch}"()
+// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b64 flat_scratch, 0", "~{flat_scratch}"()
   __asm__ volatile("s_mov_b64 flat_scratch, 0" : : : "flat_scratch");
 
-// CHECK: tail call void asm sideeffect "s_mov_b32 flat_scratch_lo, 0", "~{flat_scratch_lo}"()
+// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b32 flat_scratch_lo, 0", "~{flat_scratch_lo}"()
   __asm__ volatile("s_mov_b32 flat_scratch_lo, 0" : : : "flat_scratch_lo");
 
-// CHECK: tail call void asm sideeffect "s_mov_b32 flat_scratch_hi, 0", "~{flat_scratch_hi}"()
+// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b32 flat_scratch_hi, 0", "~{flat_scratch_hi}"()
   __asm__ volatile("s_mov_b32 flat_scratch_hi, 0" : : : "flat_scratch_hi");
 }
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index 2fda52dcd2dc6bf00dd7bfc87461a1bed866d5e0..854ab39791f1643fc650e95603574279deb09dd2 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -49,6 +49,7 @@
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1103 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1103 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1150 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1150 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1151 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1151 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1152 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1152 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1200 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1200 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1201 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1201 %s
 
@@ -100,6 +101,7 @@
 // GFX1103: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1150: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1151: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
+// GFX1152: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl
index 3c40370e7f107656387de9a44acf1aa6b8b5dd9b..f30776a8bb85b9d7fa814aed7ae9c933caa21c4f 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl
@@ -2,44 +2,45 @@
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1010 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1011 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1012 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s
 
 typedef unsigned int uint;
 typedef unsigned long ulong;
 
 // CHECK-LABEL: @test_permlane16(
-// CHECK: call i32 @llvm.amdgcn.permlane16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlane16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false)
 void test_permlane16(global uint* out, uint a, uint b, uint c, uint d) {
   *out = __builtin_amdgcn_permlane16(a, b, c, d, 0, 0);
 }
 
 // CHECK-LABEL: @test_permlanex16(
-// CHECK: call i32 @llvm.amdgcn.permlanex16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlanex16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false)
 void test_permlanex16(global uint* out, uint a, uint b, uint c, uint d) {
   *out = __builtin_amdgcn_permlanex16(a, b, c, d, 0, 0);
 }
 
 // CHECK-LABEL: @test_mov_dpp8(
-// CHECK: call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %a, i32 1)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mov.dpp8.i32(i32 %a, i32 1)
 void test_mov_dpp8(global uint* out, uint a) {
   *out = __builtin_amdgcn_mov_dpp8(a, 1);
 }
 
 // CHECK-LABEL: @test_s_memtime
-// CHECK: call i64 @llvm.amdgcn.s.memtime()
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memtime()
 void test_s_memtime(global ulong* out)
 {
   *out = __builtin_amdgcn_s_memtime();
 }
 
 // CHECK-LABEL: @test_groupstaticsize
-// CHECK: call i32 @llvm.amdgcn.groupstaticsize()
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.groupstaticsize()
 void test_groupstaticsize(global uint* out)
 {
   *out = __builtin_amdgcn_groupstaticsize();
 }
 
 // CHECK-LABEL: @test_ballot_wave32(
-// CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 %{{.+}})
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ballot.i32(i1 %{{.+}})
 void test_ballot_wave32(global uint* out, int a, int b)
 {
   *out = __builtin_amdgcn_ballot_w32(a == b);
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
index d17ff81e5d43ca04397f0fc316f943453ed0dd37..868b5bed0c9522df71a689cd6844f441d1a3e507 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
@@ -5,6 +5,8 @@
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1103 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1150 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1151 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1152 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s
 
 typedef unsigned int uint;
 typedef unsigned long ulong;
@@ -12,19 +14,19 @@ typedef uint uint2 __attribute__((ext_vector_type(2)));
 typedef uint uint4 __attribute__((ext_vector_type(4)));
 
 // CHECK-LABEL: @test_s_sendmsg_rtn(
-// CHECK: call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 0)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 0)
 void test_s_sendmsg_rtn(global uint* out) {
   *out = __builtin_amdgcn_s_sendmsg_rtn(0);
 }
 
 // CHECK-LABEL: @test_s_sendmsg_rtnl(
-// CHECK: call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 0)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 0)
 void test_s_sendmsg_rtnl(global ulong* out) {
   *out = __builtin_amdgcn_s_sendmsg_rtnl(0);
 }
 
 // CHECK-LABEL: @test_ds_bvh_stack_rtn(
-// CHECK: %0 = tail call { i32, i32 } @llvm.amdgcn.ds.bvh.stack.rtn(i32 %addr, i32 %data, <4 x i32> %data1, i32 128)
+// CHECK: %0 = tail call{{.*}} { i32, i32 } @llvm.amdgcn.ds.bvh.stack.rtn(i32 %addr, i32 %data, <4 x i32> %data1, i32 128)
 // CHECK: %1 = extractvalue { i32, i32 } %0, 0
 // CHECK: %2 = extractvalue { i32, i32 } %0, 1
 // CHECK: %3 = insertelement <2 x i32> poison, i32 %1, i64 0
@@ -35,19 +37,19 @@ void test_ds_bvh_stack_rtn(global uint2* out, uint addr, uint data, uint4 data1)
 }
 
 // CHECK-LABEL: @test_permlane64(
-// CHECK: call i32 @llvm.amdgcn.permlane64(i32 %a)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlane64(i32 %a)
 void test_permlane64(global uint* out, uint a) {
   *out = __builtin_amdgcn_permlane64(a);
 }
 
 // CHECK-LABEL: @test_s_wait_event_export_ready
-// CHECK: call void @llvm.amdgcn.s.wait.event.export.ready
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.wait.event.export.ready
 void test_s_wait_event_export_ready() {
   __builtin_amdgcn_s_wait_event_export_ready();
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
index d135d33d7dec6ea21a74e398aa3505c316e7eb53..ea2aedf8d44a5243049b5399804d5581939ff35b 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
@@ -3,6 +3,7 @@
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx900 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1010 -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1012 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
@@ -10,42 +11,42 @@ typedef unsigned long ulong;
 typedef unsigned int  uint;
 
 // CHECK-LABEL: @test_div_fixup_f16
-// CHECK: call half @llvm.amdgcn.div.fixup.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.div.fixup.f16
 void test_div_fixup_f16(global half* out, half a, half b, half c)
 {
   *out = __builtin_amdgcn_div_fixuph(a, b, c);
 }
 
 // CHECK-LABEL: @test_rcp_f16
-// CHECK: call half @llvm.amdgcn.rcp.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.rcp.f16
 void test_rcp_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_rcph(a);
 }
 
 // CHECK-LABEL: @test_sqrt_f16
-// CHECK: call half @llvm.sqrt.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.{{((amdgcn.){0,1})}}sqrt.f16
 void test_sqrt_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_sqrth(a);
 }
 
 // CHECK-LABEL: @test_rsq_f16
-// CHECK: call half @llvm.amdgcn.rsq.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.rsq.f16
 void test_rsq_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_rsqh(a);
 }
 
 // CHECK-LABEL: @test_sin_f16
-// CHECK: call half @llvm.amdgcn.sin.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.sin.f16
 void test_sin_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_sinh(a);
 }
 
 // CHECK-LABEL: @test_cos_f16
-// CHECK: call half @llvm.amdgcn.cos.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.cos.f16
 void test_cos_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_cosh(a);
@@ -53,102 +54,114 @@ void test_cos_f16(global half* out, half a)
 
 // CHECK-LABEL: @test_ldexp_f16
 // CHECK: [[TRUNC:%[0-9a-z]+]] = trunc i32
-// CHECK: call half @llvm.ldexp.f16.i16(half %a, i16 [[TRUNC]])
+// CHECK: {{.*}}call{{.*}} half @llvm.ldexp.f16.i16(half %a, i16 [[TRUNC]])
 void test_ldexp_f16(global half* out, half a, int b)
 {
   *out = __builtin_amdgcn_ldexph(a, b);
 }
 
 // CHECK-LABEL: @test_frexp_mant_f16
-// CHECK: call half @llvm.amdgcn.frexp.mant.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.frexp.mant.f16
 void test_frexp_mant_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_frexp_manth(a);
 }
 
 // CHECK-LABEL: @test_frexp_exp_f16
-// CHECK: call i16 @llvm.amdgcn.frexp.exp.i16.f16
+// CHECK: {{.*}}call{{.*}} i16 @llvm.amdgcn.frexp.exp.i16.f16
 void test_frexp_exp_f16(global short* out, half a)
 {
   *out = __builtin_amdgcn_frexp_exph(a);
 }
 
 // CHECK-LABEL: @test_fract_f16
-// CHECK: call half @llvm.amdgcn.fract.f16
+// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.fract.f16
 void test_fract_f16(global half* out, half a)
 {
   *out = __builtin_amdgcn_fracth(a);
 }
 
 // CHECK-LABEL: @test_class_f16
-// CHECK: call i1 @llvm.amdgcn.class.f16
+// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f16
 void test_class_f16(global half* out, half a, int b)
 {
   *out = __builtin_amdgcn_classh(a, b);
 }
 
 // CHECK-LABEL: @test_s_memrealtime
-// CHECK: call i64 @llvm.amdgcn.s.memrealtime()
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memrealtime()
 void test_s_memrealtime(global ulong* out)
 {
   *out = __builtin_amdgcn_s_memrealtime();
 }
 
 // CHECK-LABEL: @test_s_dcache_wb()
-// CHECK: call void @llvm.amdgcn.s.dcache.wb()
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.dcache.wb()
 void test_s_dcache_wb()
 {
   __builtin_amdgcn_s_dcache_wb();
 }
 
 // CHECK-LABEL: @test_mov_dpp
-// CHECK: call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 %src, i32 0, i32 0, i32 0, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 %src, i32 0, i32 0, i32 0, i1 false)
 void test_mov_dpp(global int* out, int src)
 {
   *out = __builtin_amdgcn_mov_dpp(src, 0, 0, 0, false);
 }
 
 // CHECK-LABEL: @test_update_dpp
-// CHECK: call i32 @llvm.amdgcn.update.dpp.i32(i32 %arg1, i32 %arg2, i32 0, i32 0, i32 0, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %arg1, i32 %arg2, i32 0, i32 0, i32 0, i1 false)
 void test_update_dpp(global int* out, int arg1, int arg2)
 {
   *out = __builtin_amdgcn_update_dpp(arg1, arg2, 0, 0, 0, false);
 }
 
 // CHECK-LABEL: @test_ds_fadd
-// CHECK: call float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+#if !defined(__SPIRV__)
 void test_ds_faddf(local float *out, float src) {
+#else
+void test_ds_faddf(__attribute__((address_space(3))) float *out, float src) {
+#endif
   *out = __builtin_amdgcn_ds_faddf(out, src, 0, 0, false);
 }
 
 // CHECK-LABEL: @test_ds_fmin
-// CHECK: call float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+#if !defined(__SPIRV__)
 void test_ds_fminf(local float *out, float src) {
+#else
+void test_ds_fminf(__attribute__((address_space(3))) float *out, float src) {
+#endif
   *out = __builtin_amdgcn_ds_fminf(out, src, 0, 0, false);
 }
 
 // CHECK-LABEL: @test_ds_fmax
-// CHECK: call float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false)
+#if !defined(__SPIRV__)
 void test_ds_fmaxf(local float *out, float src) {
+#else
+void test_ds_fmaxf(__attribute__((address_space(3))) float *out, float src) {
+#endif
   *out = __builtin_amdgcn_ds_fmaxf(out, src, 0, 0, false);
 }
 
 // CHECK-LABEL: @test_s_memtime
-// CHECK: call i64 @llvm.amdgcn.s.memtime()
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memtime()
 void test_s_memtime(global ulong* out)
 {
   *out = __builtin_amdgcn_s_memtime();
 }
 
 // CHECK-LABEL: @test_perm
-// CHECK: call i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 %s)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 %s)
 void test_perm(global uint* out, uint a, uint b, uint s)
 {
   *out = __builtin_amdgcn_perm(a, b, s);
 }
 
 // CHECK-LABEL: @test_groupstaticsize
-// CHECK: call i32 @llvm.amdgcn.groupstaticsize()
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.groupstaticsize()
 void test_groupstaticsize(global uint* out)
 {
   *out = __builtin_amdgcn_groupstaticsize();
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index c2ef9ea947e93c7d87103f5d81053cecfa26feec..ffc190b76db98b6c8b282ecee431ae025e04b11f 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -1,5 +1,7 @@
 // REQUIRES: amdgpu-registered-target
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu tahiti -emit-llvm -o - %s | FileCheck -enable-var-scope %s
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu tahiti -emit-llvm -o - %s | FileCheck -enable-var-scope --check-prefixes=CHECK-AMDGCN,CHECK %s
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck -enable-var-scope --check-prefix=CHECK %s
+
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
@@ -12,7 +14,7 @@ typedef ushort __attribute__((ext_vector_type(2))) ushort2;
 typedef uint __attribute__((ext_vector_type(4))) uint4;
 
 // CHECK-LABEL: @test_div_scale_f64
-// CHECK: call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true)
+// CHECK: {{.*}}call{{.*}} { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true)
 // CHECK-DAG: [[FLAG:%.+]] = extractvalue { double, i1 } %{{.+}}, 1
 // CHECK-DAG: [[VAL:%.+]] = extractvalue { double, i1 } %{{.+}}, 0
 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i32
@@ -25,7 +27,7 @@ void test_div_scale_f64(global double* out, global int* flagout, double a, doubl
 }
 
 // CHECK-LABEL: @test_div_scale_f32(
-// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
+// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
 // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1
 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0
 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8
@@ -38,7 +40,7 @@ void test_div_scale_f32(global float* out, global bool* flagout, float a, float
 }
 
 // CHECK-LABEL: @test_div_scale_f32_global_ptr(
-// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
+// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
 // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1
 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0
 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8
@@ -49,7 +51,7 @@ void test_div_scale_f32_global_ptr(global float* out, global int* flagout, float
 }
 
 // CHECK-LABEL: @test_div_scale_f32_generic_ptr(
-// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
+// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
 // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1
 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0
 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8
@@ -61,360 +63,360 @@ void test_div_scale_f32_generic_ptr(global float* out, global int* flagout, floa
 }
 
 // CHECK-LABEL: @test_div_fmas_f32
-// CHECK: call float @llvm.amdgcn.div.fmas.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.div.fmas.f32
 void test_div_fmas_f32(global float* out, float a, float b, float c, int d)
 {
   *out = __builtin_amdgcn_div_fmasf(a, b, c, d);
 }
 
 // CHECK-LABEL: @test_div_fmas_f64
-// CHECK: call double @llvm.amdgcn.div.fmas.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.div.fmas.f64
 void test_div_fmas_f64(global double* out, double a, double b, double c, int d)
 {
   *out = __builtin_amdgcn_div_fmas(a, b, c, d);
 }
 
 // CHECK-LABEL: @test_div_fixup_f32
-// CHECK: call float @llvm.amdgcn.div.fixup.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.div.fixup.f32
 void test_div_fixup_f32(global float* out, float a, float b, float c)
 {
   *out = __builtin_amdgcn_div_fixupf(a, b, c);
 }
 
 // CHECK-LABEL: @test_div_fixup_f64
-// CHECK: call double @llvm.amdgcn.div.fixup.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.div.fixup.f64
 void test_div_fixup_f64(global double* out, double a, double b, double c)
 {
   *out = __builtin_amdgcn_div_fixup(a, b, c);
 }
 
 // CHECK-LABEL: @test_trig_preop_f32
-// CHECK: call float @llvm.amdgcn.trig.preop.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.trig.preop.f32
 void test_trig_preop_f32(global float* out, float a, int b)
 {
   *out = __builtin_amdgcn_trig_preopf(a, b);
 }
 
 // CHECK-LABEL: @test_trig_preop_f64
-// CHECK: call double @llvm.amdgcn.trig.preop.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.trig.preop.f64
 void test_trig_preop_f64(global double* out, double a, int b)
 {
   *out = __builtin_amdgcn_trig_preop(a, b);
 }
 
 // CHECK-LABEL: @test_rcp_f32
-// CHECK: call float @llvm.amdgcn.rcp.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rcp.f32
 void test_rcp_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_rcpf(a);
 }
 
 // CHECK-LABEL: @test_rcp_f64
-// CHECK: call double @llvm.amdgcn.rcp.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rcp.f64
 void test_rcp_f64(global double* out, double a)
 {
   *out = __builtin_amdgcn_rcp(a);
 }
 
 // CHECK-LABEL: @test_sqrt_f32
-// CHECK: call float @llvm.amdgcn.sqrt.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.{{((amdgcn.){0,1})}}sqrt.f32
 void test_sqrt_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_sqrtf(a);
 }
 
 // CHECK-LABEL: @test_sqrt_f64
-// CHECK: call double @llvm.amdgcn.sqrt.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.sqrt.f64
 void test_sqrt_f64(global double* out, double a)
 {
   *out = __builtin_amdgcn_sqrt(a);
 }
 
 // CHECK-LABEL: @test_rsq_f32
-// CHECK: call float @llvm.amdgcn.rsq.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rsq.f32
 void test_rsq_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_rsqf(a);
 }
 
 // CHECK-LABEL: @test_rsq_f64
-// CHECK: call double @llvm.amdgcn.rsq.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rsq.f64
 void test_rsq_f64(global double* out, double a)
 {
   *out = __builtin_amdgcn_rsq(a);
 }
 
 // CHECK-LABEL: @test_rsq_clamp_f32
-// CHECK: call float @llvm.amdgcn.rsq.clamp.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rsq.clamp.f32
 void test_rsq_clamp_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_rsq_clampf(a);
 }
 
 // CHECK-LABEL: @test_rsq_clamp_f64
-// CHECK: call double @llvm.amdgcn.rsq.clamp.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rsq.clamp.f64
 void test_rsq_clamp_f64(global double* out, double a)
 {
   *out = __builtin_amdgcn_rsq_clamp(a);
 }
 
 // CHECK-LABEL: @test_sin_f32
-// CHECK: call float @llvm.amdgcn.sin.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.sin.f32
 void test_sin_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_sinf(a);
 }
 
 // CHECK-LABEL: @test_cos_f32
-// CHECK: call float @llvm.amdgcn.cos.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cos.f32
 void test_cos_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_cosf(a);
 }
 
 // CHECK-LABEL: @test_log_f32
-// CHECK: call float @llvm.amdgcn.log.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.log.f32
 void test_log_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_logf(a);
 }
 
 // CHECK-LABEL: @test_exp2_f32
-// CHECK: call float @llvm.amdgcn.exp2.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.exp2.f32
 void test_exp2_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_exp2f(a);
 }
 
 // CHECK-LABEL: @test_log_clamp_f32
-// CHECK: call float @llvm.amdgcn.log.clamp.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.log.clamp.f32
 void test_log_clamp_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_log_clampf(a);
 }
 
 // CHECK-LABEL: @test_ldexp_f32
-// CHECK: call float @llvm.ldexp.f32.i32
+// CHECK: {{.*}}call{{.*}} float @llvm.ldexp.f32.i32
 void test_ldexp_f32(global float* out, float a, int b)
 {
   *out = __builtin_amdgcn_ldexpf(a, b);
 }
 
 // CHECK-LABEL: @test_ldexp_f64
-// CHECK: call double @llvm.ldexp.f64.i32
+// CHECK: {{.*}}call{{.*}} double @llvm.ldexp.f64.i32
 void test_ldexp_f64(global double* out, double a, int b)
 {
   *out = __builtin_amdgcn_ldexp(a, b);
 }
 
 // CHECK-LABEL: @test_frexp_mant_f32
-// CHECK: call float @llvm.amdgcn.frexp.mant.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.frexp.mant.f32
 void test_frexp_mant_f32(global float* out, float a)
 {
   *out = __builtin_amdgcn_frexp_mantf(a);
 }
 
 // CHECK-LABEL: @test_frexp_mant_f64
-// CHECK: call double @llvm.amdgcn.frexp.mant.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.frexp.mant.f64
 void test_frexp_mant_f64(global double* out, double a)
 {
   *out = __builtin_amdgcn_frexp_mant(a);
 }
 
 // CHECK-LABEL: @test_frexp_exp_f32
-// CHECK: call i32 @llvm.amdgcn.frexp.exp.i32.f32
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.frexp.exp.i32.f32
 void test_frexp_exp_f32(global int* out, float a)
 {
   *out = __builtin_amdgcn_frexp_expf(a);
 }
 
 // CHECK-LABEL: @test_frexp_exp_f64
-// CHECK: call i32 @llvm.amdgcn.frexp.exp.i32.f64
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.frexp.exp.i32.f64
 void test_frexp_exp_f64(global int* out, double a)
 {
   *out = __builtin_amdgcn_frexp_exp(a);
 }
 
 // CHECK-LABEL: @test_fract_f32
-// CHECK: call float @llvm.amdgcn.fract.f32
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.fract.f32
 void test_fract_f32(global int* out, float a)
 {
   *out = __builtin_amdgcn_fractf(a);
 }
 
 // CHECK-LABEL: @test_fract_f64
-// CHECK: call double @llvm.amdgcn.fract.f64
+// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.fract.f64
 void test_fract_f64(global int* out, double a)
 {
   *out = __builtin_amdgcn_fract(a);
 }
 
 // CHECK-LABEL: @test_lerp
-// CHECK: call i32 @llvm.amdgcn.lerp
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.lerp
 void test_lerp(global int* out, int a, int b, int c)
 {
   *out = __builtin_amdgcn_lerp(a, b, c);
 }
 
 // CHECK-LABEL: @test_sicmp_i32
-// CHECK: call i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32)
 void test_sicmp_i32(global ulong* out, int a, int b)
 {
   *out = __builtin_amdgcn_sicmp(a, b, 32);
 }
 
 // CHECK-LABEL: @test_uicmp_i32
-// CHECK: call i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32)
 void test_uicmp_i32(global ulong* out, uint a, uint b)
 {
   *out = __builtin_amdgcn_uicmp(a, b, 32);
 }
 
 // CHECK-LABEL: @test_sicmp_i64
-// CHECK: call i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 38)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 38)
 void test_sicmp_i64(global ulong* out, long a, long b)
 {
   *out = __builtin_amdgcn_sicmpl(a, b, 39-1);
 }
 
 // CHECK-LABEL: @test_uicmp_i64
-// CHECK: call i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 35)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 35)
 void test_uicmp_i64(global ulong* out, ulong a, ulong b)
 {
   *out = __builtin_amdgcn_uicmpl(a, b, 30+5);
 }
 
 // CHECK-LABEL: @test_ds_swizzle
-// CHECK: call i32 @llvm.amdgcn.ds.swizzle(i32 %a, i32 32)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.swizzle(i32 %a, i32 32)
 void test_ds_swizzle(global int* out, int a)
 {
   *out = __builtin_amdgcn_ds_swizzle(a, 32);
 }
 
 // CHECK-LABEL: @test_ds_permute
-// CHECK: call i32 @llvm.amdgcn.ds.permute(i32 %a, i32 %b)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.permute(i32 %a, i32 %b)
 void test_ds_permute(global int* out, int a, int b)
 {
   out[0] = __builtin_amdgcn_ds_permute(a, b);
 }
 
 // CHECK-LABEL: @test_ds_bpermute
-// CHECK: call i32 @llvm.amdgcn.ds.bpermute(i32 %a, i32 %b)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.bpermute(i32 %a, i32 %b)
 void test_ds_bpermute(global int* out, int a, int b)
 {
   *out = __builtin_amdgcn_ds_bpermute(a, b);
 }
 
 // CHECK-LABEL: @test_readfirstlane
-// CHECK: call i32 @llvm.amdgcn.readfirstlane(i32 %a)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.readfirstlane(i32 %a)
 void test_readfirstlane(global int* out, int a)
 {
   *out = __builtin_amdgcn_readfirstlane(a);
 }
 
 // CHECK-LABEL: @test_readlane
-// CHECK: call i32 @llvm.amdgcn.readlane(i32 %a, i32 %b)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.readlane(i32 %a, i32 %b)
 void test_readlane(global int* out, int a, int b)
 {
   *out = __builtin_amdgcn_readlane(a, b);
 }
 
 // CHECK-LABEL: @test_fcmp_f32
-// CHECK: call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 5)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 5)
 void test_fcmp_f32(global ulong* out, float a, float b)
 {
   *out = __builtin_amdgcn_fcmpf(a, b, 5);
 }
 
 // CHECK-LABEL: @test_fcmp_f64
-// CHECK: call i64 @llvm.amdgcn.fcmp.i64.f64(double %a, double %b, i32 6)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.fcmp.i64.f64(double %a, double %b, i32 6)
 void test_fcmp_f64(global ulong* out, double a, double b)
 {
   *out = __builtin_amdgcn_fcmp(a, b, 3+3);
 }
 
 // CHECK-LABEL: @test_class_f32
-// CHECK: call i1 @llvm.amdgcn.class.f32
+// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f32
 void test_class_f32(global float* out, float a, int b)
 {
   *out = __builtin_amdgcn_classf(a, b);
 }
 
 // CHECK-LABEL: @test_class_f64
-// CHECK: call i1 @llvm.amdgcn.class.f64
+// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f64
 void test_class_f64(global double* out, double a, int b)
 {
   *out = __builtin_amdgcn_class(a, b);
 }
 
 // CHECK-LABEL: @test_buffer_wbinvl1
-// CHECK: call void @llvm.amdgcn.buffer.wbinvl1(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.buffer.wbinvl1(
 void test_buffer_wbinvl1()
 {
   __builtin_amdgcn_buffer_wbinvl1();
 }
 
 // CHECK-LABEL: @test_s_dcache_inv
-// CHECK: call void @llvm.amdgcn.s.dcache.inv(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.dcache.inv(
 void test_s_dcache_inv()
 {
   __builtin_amdgcn_s_dcache_inv();
 }
 
 // CHECK-LABEL: @test_s_waitcnt
-// CHECK: call void @llvm.amdgcn.s.waitcnt(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.waitcnt(
 void test_s_waitcnt()
 {
   __builtin_amdgcn_s_waitcnt(0);
 }
 
 // CHECK-LABEL: @test_s_sendmsg
-// CHECK: call void @llvm.amdgcn.s.sendmsg(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsg(
 void test_s_sendmsg()
 {
   __builtin_amdgcn_s_sendmsg(1, 0);
 }
 
 // CHECK-LABEL: @test_s_sendmsg_var
-// CHECK: call void @llvm.amdgcn.s.sendmsg(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsg(
 void test_s_sendmsg_var(int in)
 {
   __builtin_amdgcn_s_sendmsg(1, in);
 }
 
 // CHECK-LABEL: @test_s_sendmsghalt
-// CHECK: call void @llvm.amdgcn.s.sendmsghalt(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsghalt(
 void test_s_sendmsghalt()
 {
   __builtin_amdgcn_s_sendmsghalt(1, 0);
 }
 
 // CHECK-LABEL: @test_s_sendmsghalt
-// CHECK: call void @llvm.amdgcn.s.sendmsghalt(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsghalt(
 void test_s_sendmsghalt_var(int in)
 {
   __builtin_amdgcn_s_sendmsghalt(1, in);
 }
 
 // CHECK-LABEL: @test_s_barrier
-// CHECK: call void @llvm.amdgcn.s.barrier(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.barrier(
 void test_s_barrier()
 {
   __builtin_amdgcn_s_barrier();
 }
 
 // CHECK-LABEL: @test_wave_barrier
-// CHECK: call void @llvm.amdgcn.wave.barrier(
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.wave.barrier(
 void test_wave_barrier()
 {
   __builtin_amdgcn_wave_barrier();
 }
 
 // CHECK-LABEL: @test_sched_barrier
-// CHECK: call void @llvm.amdgcn.sched.barrier(i32 0)
-// CHECK: call void @llvm.amdgcn.sched.barrier(i32 1)
-// CHECK: call void @llvm.amdgcn.sched.barrier(i32 4)
-// CHECK: call void @llvm.amdgcn.sched.barrier(i32 15)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 0)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 4)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 15)
 void test_sched_barrier()
 {
   __builtin_amdgcn_sched_barrier(0);
@@ -424,10 +426,10 @@ void test_sched_barrier()
 }
 
 // CHECK-LABEL: @test_sched_group_barrier
-// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 0, i32 1, i32 2)
-// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 1, i32 2, i32 4)
-// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 4, i32 8, i32 16)
-// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 15, i32 10000, i32 -1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 0, i32 1, i32 2)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 1, i32 2, i32 4)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 4, i32 8, i32 16)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 15, i32 10000, i32 -1)
 void test_sched_group_barrier()
 {
   __builtin_amdgcn_sched_group_barrier(0, 1, 2);
@@ -437,10 +439,10 @@ void test_sched_group_barrier()
 }
 
 // CHECK-LABEL: @test_iglp_opt
-// CHECK: call void @llvm.amdgcn.iglp.opt(i32 0)
-// CHECK: call void @llvm.amdgcn.iglp.opt(i32 1)
-// CHECK: call void @llvm.amdgcn.iglp.opt(i32 4)
-// CHECK: call void @llvm.amdgcn.iglp.opt(i32 15)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 0)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 4)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 15)
 void test_iglp_opt()
 {
   __builtin_amdgcn_iglp_opt(0);
@@ -450,8 +452,8 @@ void test_iglp_opt()
 }
 
 // CHECK-LABEL: @test_s_sleep
-// CHECK: call void @llvm.amdgcn.s.sleep(i32 1)
-// CHECK: call void @llvm.amdgcn.s.sleep(i32 15)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sleep(i32 1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sleep(i32 15)
 void test_s_sleep()
 {
   __builtin_amdgcn_s_sleep(1);
@@ -459,8 +461,8 @@ void test_s_sleep()
 }
 
 // CHECK-LABEL: @test_s_incperflevel
-// CHECK: call void @llvm.amdgcn.s.incperflevel(i32 1)
-// CHECK: call void @llvm.amdgcn.s.incperflevel(i32 15)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.incperflevel(i32 1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.incperflevel(i32 15)
 void test_s_incperflevel()
 {
   __builtin_amdgcn_s_incperflevel(1);
@@ -468,8 +470,8 @@ void test_s_incperflevel()
 }
 
 // CHECK-LABEL: @test_s_decperflevel
-// CHECK: call void @llvm.amdgcn.s.decperflevel(i32 1)
-// CHECK: call void @llvm.amdgcn.s.decperflevel(i32 15)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.decperflevel(i32 1)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.decperflevel(i32 15)
 void test_s_decperflevel()
 {
   __builtin_amdgcn_s_decperflevel(1);
@@ -477,8 +479,8 @@ void test_s_decperflevel()
 }
 
 // CHECK-LABEL: @test_s_setprio
-// CHECK: call void @llvm.amdgcn.s.setprio(i16 0)
-// CHECK: call void @llvm.amdgcn.s.setprio(i16 3)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setprio(i16 0)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setprio(i16 3)
 void test_s_setprio()
 {
   __builtin_amdgcn_s_setprio(0);
@@ -486,47 +488,47 @@ void test_s_setprio()
 }
 
 // CHECK-LABEL: @test_cubeid(
-// CHECK: call float @llvm.amdgcn.cubeid(float %a, float %b, float %c)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubeid(float %a, float %b, float %c)
 void test_cubeid(global float* out, float a, float b, float c) {
   *out = __builtin_amdgcn_cubeid(a, b, c);
 }
 
 // CHECK-LABEL: @test_cubesc(
-// CHECK: call float @llvm.amdgcn.cubesc(float %a, float %b, float %c)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubesc(float %a, float %b, float %c)
 void test_cubesc(global float* out, float a, float b, float c) {
   *out = __builtin_amdgcn_cubesc(a, b, c);
 }
 
 // CHECK-LABEL: @test_cubetc(
-// CHECK: call float @llvm.amdgcn.cubetc(float %a, float %b, float %c)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubetc(float %a, float %b, float %c)
 void test_cubetc(global float* out, float a, float b, float c) {
   *out = __builtin_amdgcn_cubetc(a, b, c);
 }
 
 // CHECK-LABEL: @test_cubema(
-// CHECK: call float @llvm.amdgcn.cubema(float %a, float %b, float %c)
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubema(float %a, float %b, float %c)
 void test_cubema(global float* out, float a, float b, float c) {
   *out = __builtin_amdgcn_cubema(a, b, c);
 }
 
 // CHECK-LABEL: @test_read_exec(
-// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.ballot.i64(i1 true)
 void test_read_exec(global ulong* out) {
   *out = __builtin_amdgcn_read_exec();
 }
 
-// CHECK: declare i64 @llvm.amdgcn.ballot.i64(i1) #[[$NOUNWIND_READONLY:[0-9]+]]
+// CHECK: declare i64 @llvm.amdgcn.ballot.i64(i1){{.*}} #[[$NOUNWIND_READONLY:[0-9]+]]
 
 // CHECK-LABEL: @test_read_exec_lo(
-// CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 true)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ballot.i32(i1 true)
 void test_read_exec_lo(global uint* out) {
   *out = __builtin_amdgcn_read_exec_lo();
 }
 
-// CHECK: declare i32 @llvm.amdgcn.ballot.i32(i1) #[[$NOUNWIND_READONLY:[0-9]+]]
+// CHECK: declare i32 @llvm.amdgcn.ballot.i32(i1){{.*}} #[[$NOUNWIND_READONLY:[0-9]+]]
 
 // CHECK-LABEL: @test_read_exec_hi(
-// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.ballot.i64(i1 true)
 // CHECK: lshr i64 [[A:%.*]], 32
 // CHECK: trunc nuw i64 [[B:%.*]] to i32
 void test_read_exec_hi(global uint* out) {
@@ -534,37 +536,53 @@ void test_read_exec_hi(global uint* out) {
 }
 
 // CHECK-LABEL: @test_dispatch_ptr
-// CHECK: call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// CHECK: {{.*}}call align 4 dereferenceable(64){{.*}} ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+#if !defined(__SPIRV__)
 void test_dispatch_ptr(__constant unsigned char ** out)
+#else
+void test_dispatch_ptr(__attribute__((address_space(4))) unsigned char ** out)
+#endif
 {
   *out = __builtin_amdgcn_dispatch_ptr();
 }
 
 // CHECK-LABEL: @test_queue_ptr
-// CHECK: call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
+// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.queue.ptr()
+#if !defined(__SPIRV__)
 void test_queue_ptr(__constant unsigned char ** out)
+#else
+void test_queue_ptr(__attribute__((address_space(4))) unsigned char ** out)
+#endif
 {
   *out = __builtin_amdgcn_queue_ptr();
 }
 
 // CHECK-LABEL: @test_kernarg_segment_ptr
-// CHECK: call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+#if !defined(__SPIRV__)
 void test_kernarg_segment_ptr(__constant unsigned char ** out)
+#else
+void test_kernarg_segment_ptr(__attribute__((address_space(4))) unsigned char ** out)
+#endif
 {
   *out = __builtin_amdgcn_kernarg_segment_ptr();
 }
 
 // CHECK-LABEL: @test_implicitarg_ptr
-// CHECK: call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+#if !defined(__SPIRV__)
 void test_implicitarg_ptr(__constant unsigned char ** out)
+#else
+void test_implicitarg_ptr(__attribute__((address_space(4))) unsigned char ** out)
+#endif
 {
   *out = __builtin_amdgcn_implicitarg_ptr();
 }
 
 // CHECK-LABEL: @test_get_group_id(
-// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.x()
-// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.y()
-// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.z()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.x()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.y()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.z()
 void test_get_group_id(int d, global int *out)
 {
 	switch (d) {
@@ -576,9 +594,9 @@ void test_get_group_id(int d, global int *out)
 }
 
 // CHECK-LABEL: @test_s_getreg(
-// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 0)
-// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 1)
-// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 65535)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 0)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 1)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 65535)
 void test_s_getreg(volatile global uint *out)
 {
   *out = __builtin_amdgcn_s_getreg(0);
@@ -587,9 +605,9 @@ void test_s_getreg(volatile global uint *out)
 }
 
 // CHECK-LABEL: @test_get_local_id(
-// CHECK: tail call i32 @llvm.amdgcn.workitem.id.x(), !range [[$WI_RANGE:![0-9]*]], !noundef
-// CHECK: tail call i32 @llvm.amdgcn.workitem.id.y(), !range [[$WI_RANGE]], !noundef
-// CHECK: tail call i32 @llvm.amdgcn.workitem.id.z(), !range [[$WI_RANGE]], !noundef
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.x(), !range [[$WI_RANGE:![0-9]*]], !noundef
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.y(), !range [[$WI_RANGE]], !noundef
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.z(), !range [[$WI_RANGE]], !noundef
 void test_get_local_id(int d, global int *out)
 {
 	switch (d) {
@@ -601,7 +619,7 @@ void test_get_local_id(int d, global int *out)
 }
 
 // CHECK-LABEL: @test_get_workgroup_size(
-// CHECK: call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// CHECK: {{.*}}call align 8 dereferenceable(256){{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
 // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 12
 // CHECK: load i16, ptr addrspace(4) %{{.*}}, align 4, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef
 // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 14
@@ -619,7 +637,7 @@ void test_get_workgroup_size(int d, global int *out)
 }
 
 // CHECK-LABEL: @test_get_grid_size(
-// CHECK: call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// CHECK: {{.*}}call align 4 dereferenceable(64){{.*}} ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
 // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 12
 // CHECK: load i32, ptr addrspace(4) %{{.*}}, align 4, !invariant.load
 // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 16
@@ -637,177 +655,185 @@ void test_get_grid_size(int d, global int *out)
 }
 
 // CHECK-LABEL: @test_fmed3_f32
-// CHECK: call float @llvm.amdgcn.fmed3.f32(
+// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.fmed3.f32(
 void test_fmed3_f32(global float* out, float a, float b, float c)
 {
   *out = __builtin_amdgcn_fmed3f(a, b, c);
 }
 
 // CHECK-LABEL: @test_s_getpc
-// CHECK: call i64 @llvm.amdgcn.s.getpc()
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.getpc()
 void test_s_getpc(global ulong* out)
 {
   *out = __builtin_amdgcn_s_getpc();
 }
 
 // CHECK-LABEL: @test_ds_append_lds(
-// CHECK: call i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false)
 kernel void test_ds_append_lds(global int* out, local int* ptr) {
+#if !defined(__SPIRV__)
   *out = __builtin_amdgcn_ds_append(ptr);
+#else
+  *out = __builtin_amdgcn_ds_append((__attribute__((address_space(3))) int*)(int*)ptr);
+#endif
 }
 
 // CHECK-LABEL: @test_ds_consume_lds(
-// CHECK: call i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false)
 kernel void test_ds_consume_lds(global int* out, local int* ptr) {
+#if !defined(__SPIRV__)
   *out = __builtin_amdgcn_ds_consume(ptr);
+#else
+  *out = __builtin_amdgcn_ds_consume((__attribute__((address_space(3))) int*)(int*)ptr);
+#endif
 }
 
 // CHECK-LABEL: @test_gws_init(
-// CHECK: call void @llvm.amdgcn.ds.gws.init(i32 %value, i32 %id)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.init(i32 %value, i32 %id)
 kernel void test_gws_init(uint value, uint id) {
   __builtin_amdgcn_ds_gws_init(value, id);
 }
 
 // CHECK-LABEL: @test_gws_barrier(
-// CHECK: call void @llvm.amdgcn.ds.gws.barrier(i32 %value, i32 %id)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.barrier(i32 %value, i32 %id)
 kernel void test_gws_barrier(uint value, uint id) {
   __builtin_amdgcn_ds_gws_barrier(value, id);
 }
 
 // CHECK-LABEL: @test_gws_sema_v(
-// CHECK: call void @llvm.amdgcn.ds.gws.sema.v(i32 %id)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.v(i32 %id)
 kernel void test_gws_sema_v(uint id) {
   __builtin_amdgcn_ds_gws_sema_v(id);
 }
 
 // CHECK-LABEL: @test_gws_sema_br(
-// CHECK: call void @llvm.amdgcn.ds.gws.sema.br(i32 %value, i32 %id)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.br(i32 %value, i32 %id)
 kernel void test_gws_sema_br(uint value, uint id) {
   __builtin_amdgcn_ds_gws_sema_br(value, id);
 }
 
 // CHECK-LABEL: @test_gws_sema_p(
-// CHECK: call void @llvm.amdgcn.ds.gws.sema.p(i32 %id)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.p(i32 %id)
 kernel void test_gws_sema_p(uint id) {
   __builtin_amdgcn_ds_gws_sema_p(id);
 }
 
 // CHECK-LABEL: @test_mbcnt_lo(
-// CHECK: call i32 @llvm.amdgcn.mbcnt.lo(i32 %src0, i32 %src1)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mbcnt.lo(i32 %src0, i32 %src1)
 kernel void test_mbcnt_lo(global uint* out, uint src0, uint src1) {
   *out = __builtin_amdgcn_mbcnt_lo(src0, src1);
 }
 
 // CHECK-LABEL: @test_mbcnt_hi(
-// CHECK: call i32 @llvm.amdgcn.mbcnt.hi(i32 %src0, i32 %src1)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mbcnt.hi(i32 %src0, i32 %src1)
 kernel void test_mbcnt_hi(global uint* out, uint src0, uint src1) {
   *out = __builtin_amdgcn_mbcnt_hi(src0, src1);
 }
 
 // CHECK-LABEL: @test_alignbit(
-// CHECK: tail call i32 @llvm.fshr.i32(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.fshr.i32(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_alignbit(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_alignbit(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_alignbyte(
-// CHECK: tail call i32 @llvm.amdgcn.alignbyte(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.alignbyte(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_alignbyte(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_alignbyte(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_ubfe(
-// CHECK: tail call i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_ubfe(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_ubfe(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_sbfe(
-// CHECK: tail call i32 @llvm.amdgcn.sbfe.i32(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.sbfe.i32(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_sbfe(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_sbfe(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_cvt_pkrtz(
-// CHECK: tail call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %src0, float %src1)
+// CHECK: tail call{{.*}} <2 x half> @llvm.amdgcn.cvt.pkrtz(float %src0, float %src1)
 kernel void test_cvt_pkrtz(global half2* out, float src0, float src1) {
   *out = __builtin_amdgcn_cvt_pkrtz(src0, src1);
 }
 
 // CHECK-LABEL: @test_cvt_pknorm_i16(
-// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %src0, float %src1)
+// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %src0, float %src1)
 kernel void test_cvt_pknorm_i16(global short2* out, float src0, float src1) {
   *out = __builtin_amdgcn_cvt_pknorm_i16(src0, src1);
 }
 
 // CHECK-LABEL: @test_cvt_pknorm_u16(
-// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %src0, float %src1)
+// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %src0, float %src1)
 kernel void test_cvt_pknorm_u16(global ushort2* out, float src0, float src1) {
   *out = __builtin_amdgcn_cvt_pknorm_u16(src0, src1);
 }
 
 // CHECK-LABEL: @test_cvt_pk_i16(
-// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %src0, i32 %src1)
+// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %src0, i32 %src1)
 kernel void test_cvt_pk_i16(global short2* out, int src0, int src1) {
   *out = __builtin_amdgcn_cvt_pk_i16(src0, src1);
 }
 
 // CHECK-LABEL: @test_cvt_pk_u16(
-// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %src0, i32 %src1)
+// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %src0, i32 %src1)
 kernel void test_cvt_pk_u16(global ushort2* out, uint src0, uint src1) {
   *out = __builtin_amdgcn_cvt_pk_u16(src0, src1);
 }
 
 // CHECK-LABEL: @test_cvt_pk_u8_f32
-// CHECK: tail call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src0, i32 %src1, i32 %src2)
 kernel void test_cvt_pk_u8_f32(global uint* out, float src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_cvt_pk_u8_f32(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_sad_u8(
-// CHECK: tail call i32 @llvm.amdgcn.sad.u8(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.sad.u8(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_sad_u8(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_sad_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: test_msad_u8(
-// CHECK: call i32 @llvm.amdgcn.msad.u8(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.msad.u8(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_msad_u8(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_msad_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: test_sad_hi_u8(
-// CHECK: call i32 @llvm.amdgcn.sad.hi.u8(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.sad.hi.u8(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_sad_hi_u8(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_sad_hi_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_sad_u16(
-// CHECK: call i32 @llvm.amdgcn.sad.u16(i32 %src0, i32 %src1, i32 %src2)
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.sad.u16(i32 %src0, i32 %src1, i32 %src2)
 kernel void test_sad_u16(global uint* out, uint src0, uint src1, uint src2) {
   *out = __builtin_amdgcn_sad_u16(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_qsad_pk_u16_u8(
-// CHECK: call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2)
 kernel void test_qsad_pk_u16_u8(global ulong* out, ulong src0, uint src1, ulong src2) {
   *out = __builtin_amdgcn_qsad_pk_u16_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: @test_mqsad_pk_u16_u8(
-// CHECK: call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2)
+// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2)
 kernel void test_mqsad_pk_u16_u8(global ulong* out, ulong src0, uint src1, ulong src2) {
   *out = __builtin_amdgcn_mqsad_pk_u16_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: test_mqsad_u32_u8(
-// CHECK: call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src0, i32 %src1, <4 x i32> %src2)
+// CHECK: {{.*}}call{{.*}} <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src0, i32 %src1, <4 x i32> %src2)
 kernel void test_mqsad_u32_u8(global uint4* out, ulong src0, uint src1, uint4 src2) {
   *out = __builtin_amdgcn_mqsad_u32_u8(src0, src1, src2);
 }
 
 // CHECK-LABEL: test_s_setreg(
-// CHECK: call void @llvm.amdgcn.s.setreg(i32 8193, i32 %val)
+// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setreg(i32 8193, i32 %val)
 kernel void test_s_setreg(uint val) {
   __builtin_amdgcn_s_setreg(8193, val);
 }
@@ -835,31 +861,33 @@ void test_atomic_inc_dec(local uint *lptr, global uint *gptr, uint val) {
 // CHECK-LABEL test_wavefrontsize(
 unsigned test_wavefrontsize() {
 
-  // CHECK: call i32 @llvm.amdgcn.wavefrontsize()
+  // CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.wavefrontsize()
   return __builtin_amdgcn_wavefrontsize();
 }
 
 // CHECK-LABEL test_flt_rounds(
 unsigned test_flt_rounds() {
 
-  // CHECK: call i32 @llvm.get.rounding()
+  // CHECK: {{.*}}call{{.*}} i32 @llvm.get.rounding()
   unsigned mode = __builtin_flt_rounds();
 
-  // CHECK: call void @llvm.set.rounding(i32 %0)
+#if !defined(__SPIRV__)
+  // CHECK-AMDGCN: call void @llvm.set.rounding(i32 %0)
   __builtin_set_flt_rounds(mode);
+#endif
 
   return mode;
 }
 
 // CHECK-LABEL test_get_fpenv(
 unsigned long test_get_fpenv() {
-  // CHECK: call i64 @llvm.get.fpenv.i64()
+  // CHECK: {{.*}}call{{.*}} i64 @llvm.get.fpenv.i64()
   return __builtin_amdgcn_get_fpenv();
 }
 
 // CHECK-LABEL test_set_fpenv(
 void test_set_fpenv(unsigned long env) {
-  // CHECK: call void @llvm.set.fpenv.i64(i64 %[[ENV:.+]])
+  // CHECK: {{.*}}call{{.*}} void @llvm.set.fpenv.i64(i64 %[[ENV:.+]])
   __builtin_amdgcn_set_fpenv(env);
 }
 
diff --git a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
index 259c12384f2c8d33a41c08d9b2b6040347f9082b..5ebb0ea0c33c3e1742aed92d8eafe7a8577109a5 100644
--- a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
@@ -1,11 +1,12 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -emit-llvm -O0 -o - -triple amdgcn %s | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -O0 -o - -triple spirv64-amd-amdhsa %s | FileCheck %s
 
 typedef float float32 __attribute__((ext_vector_type(32)));
 
 kernel void test_long(int arg0) {
   long v15_16;
-  // CHECK: call i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v"
+  // CHECK: call{{.*}} i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v"
   __asm volatile("v_lshlrev_b64 v[15:16], 0, %0" : "={v[15:16]}"(v15_16) : "v"(arg0));
 }
 
@@ -14,7 +15,7 @@ kernel void test_agpr() {
   float reg_a;
   float reg_b;
   float32 reg_c;
-  // CHECK:  call <32 x float> asm "v_mfma_f32_32x32x1f32 $0, $1, $2, $3", "=a,v,v,a,~{a0},~{a1},~{a2},~{a3},~{a4},~{a5},~{a6},~{a7},~{a8},~{a9},~{a10},~{a11},~{a12},~{a13},~{a14},~{a15},~{a16},~{a17},~{a18},~{a19},~{a20},~{a21},~{a22},~{a23},~{a24},~{a25},~{a26},~{a27},~{a28},~{a29},~{a30},~{a31}"
+  // CHECK:  call{{.*}} <32 x float> asm "v_mfma_f32_32x32x1f32 $0, $1, $2, $3", "=a,v,v,a,~{a0},~{a1},~{a2},~{a3},~{a4},~{a5},~{a6},~{a7},~{a8},~{a9},~{a10},~{a11},~{a12},~{a13},~{a14},~{a15},~{a16},~{a17},~{a18},~{a19},~{a20},~{a21},~{a22},~{a23},~{a24},~{a25},~{a26},~{a27},~{a28},~{a29},~{a30},~{a31}"
   __asm ("v_mfma_f32_32x32x1f32 %0, %1, %2, %3"
          : "=a"(acc_c)
          : "v"(reg_a), "v"(reg_b), "a"(reg_c)
@@ -23,12 +24,12 @@ kernel void test_agpr() {
            "a16", "a17", "a18", "a19", "a20", "a21", "a22", "a23",
            "a24", "a25", "a26", "a27", "a28", "a29", "a30", "a31");
 
-   // CHECK: call <32 x float> asm sideeffect "v_mfma_f32_32x32x1f32 a[0:31], $0, $1, a[0:31]", "={a[0:31]},v,v,{a[0:31]}"
+   // CHECK: call{{.*}} <32 x float> asm sideeffect "v_mfma_f32_32x32x1f32 a[0:31], $0, $1, a[0:31]", "={a[0:31]},v,v,{a[0:31]}"
   __asm volatile("v_mfma_f32_32x32x1f32 a[0:31], %0, %1, a[0:31]"
                  : "={a[0:31]}"(acc_c)
                  : "v"(reg_a),"v"(reg_b), "{a[0:31]}"(reg_c));
 
-  // CHECK: call float asm "v_accvgpr_read_b32 $0, $1", "={a1},{a1}"
+  // CHECK: call{{.*}} float asm "v_accvgpr_read_b32 $0, $1", "={a1},{a1}"
   __asm ("v_accvgpr_read_b32 %0, %1"
          : "={a1}"(reg_a)
          : "{a1}"(reg_b));
@@ -37,13 +38,13 @@ kernel void test_agpr() {
 kernel void test_constraint_DA() {
   const long x = 0x200000001;
   int res;
-  // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DA"(i64 8589934593)
+  // CHECK: call{{.*}} i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DA"(i64 8589934593)
   __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DA"(x));
 }
 
 kernel void test_constraint_DB() {
   const long x = 0x200000001;
   int res;
-  // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DB"(i64 8589934593)
+  // CHECK: call{{.*}} i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DB"(i64 8589934593)
   __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DB"(x));
 }
diff --git a/clang/test/Driver/aarch64-oryon-1.c b/clang/test/Driver/aarch64-oryon-1.c
new file mode 100644
index 0000000000000000000000000000000000000000..952ba5df74bafe7a29da6735926d7711022d215f
--- /dev/null
+++ b/clang/test/Driver/aarch64-oryon-1.c
@@ -0,0 +1,19 @@
+// RUN: %clang -target aarch64 -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s
+// RUN: %clang -target aarch64 -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s
+// RUN: %clang -target aarch64_be -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s
+// RUN: %clang -target aarch64 -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s
+// RUN: %clang -target aarch64 -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s
+// RUN: %clang -target aarch64_be -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s
+// Phoenix: "-cc1"{{.*}} "-triple" "aarch64{{(--)?}}"{{.*}} "-target-cpu" "oryon-1" "-target-feature" "+v8.6a"
+// Phoenix-TUNE: "-cc1"{{.*}} "-triple" "aarch64{{(--)?}}"{{.*}} "-target-cpu" "generic"
+
+// RUN: %clang -target arm64 -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix %s
+// RUN: %clang -target arm64 -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix %s
+// RUN: %clang -target arm64 -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix-TUNE %s
+// RUN: %clang -target arm64 -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix-TUNE %s
+// ARM64-Phoenix: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "oryon-1" "-target-feature" "+v8.6a"
+// ARM64-Phoenix-TUNE: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "generic"
+
+// RUN: %clang -target aarch64 -mcpu=oryon-1 -mtune=cortex-a53 -### -c %s 2>&1 | FileCheck -check-prefix=MCPU-MTUNE-Phoenix %s
+// RUN: %clang -target aarch64 -mtune=cortex-a53 -mcpu=oryon-1  -### -c %s 2>&1 | FileCheck -check-prefix=MCPU-MTUNE-Phoenix %s
+// MCPU-MTUNE-Phoenix: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "oryon-1"
diff --git a/clang/test/Driver/amdgpu-macros.cl b/clang/test/Driver/amdgpu-macros.cl
index a878a7decee95e5e4ddd0b4a71a03cb95fa12e62..3e4a570671babed9797f37d7f27693b5af03c0c5 100644
--- a/clang/test/Driver/amdgpu-macros.cl
+++ b/clang/test/Driver/amdgpu-macros.cl
@@ -127,6 +127,7 @@
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1103 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1103 -DFAMILY=GFX11
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1150 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1150 -DFAMILY=GFX11
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1151 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1151 -DFAMILY=GFX11
+// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1152 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1152 -DFAMILY=GFX11
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1200 -DFAMILY=GFX12
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1201 -DFAMILY=GFX12
 
diff --git a/clang/test/Driver/amdgpu-mcpu.cl b/clang/test/Driver/amdgpu-mcpu.cl
index 5b6a22016f043b3d0f3e8a4033175f039f28507a..4b0ef92b682a501d95eb92cfe5a166ca31afd6b1 100644
--- a/clang/test/Driver/amdgpu-mcpu.cl
+++ b/clang/test/Driver/amdgpu-mcpu.cl
@@ -112,6 +112,7 @@
 // RUN: %clang -### -target amdgcn -mcpu=gfx1103 %s 2>&1 | FileCheck --check-prefix=GFX1103 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1150 %s 2>&1 | FileCheck --check-prefix=GFX1150 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1151 %s 2>&1 | FileCheck --check-prefix=GFX1151 %s
+// RUN: %clang -### -target amdgcn -mcpu=gfx1152 %s 2>&1 | FileCheck --check-prefix=GFX1152 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX1200 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefix=GFX1201 %s
 
@@ -164,6 +165,7 @@
 // GFX1103:   "-target-cpu" "gfx1103"
 // GFX1150:   "-target-cpu" "gfx1150"
 // GFX1151:   "-target-cpu" "gfx1151"
+// GFX1152:   "-target-cpu" "gfx1152"
 // GFX1200:   "-target-cpu" "gfx1200"
 // GFX1201:   "-target-cpu" "gfx1201"
 
diff --git a/clang/test/Driver/amdgpu-openmp-toolchain.c b/clang/test/Driver/amdgpu-openmp-toolchain.c
index ef58c2c4e3f3ae24fb8105e5d4135432fd3dfcd0..49af04acc46397b48c6646c9241c6e185d26f18e 100644
--- a/clang/test/Driver/amdgpu-openmp-toolchain.c
+++ b/clang/test/Driver/amdgpu-openmp-toolchain.c
@@ -7,7 +7,7 @@
 
 // verify the tools invocations
 // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-llvm-bc"{{.*}}"-x" "c"
-// CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "gfx906"{{.*}}"-fcuda-is-device"{{.*}}
+// CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-fcuda-is-device"{{.*}}"-target-cpu" "gfx906"
 // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj"
 // CHECK: clang-linker-wrapper{{.*}} "-o" "a.out"
 
@@ -63,6 +63,7 @@
 
 // RUN: %clang -### -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a:sramecc-:xnack+ \
 // RUN:   -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-TARGET-ID
+// CHECK-TARGET-ID: "-cc1" "-triple" "amdgcn-amd-amdhsa" {{.*}} "-target-cpu" "gfx90a" "-target-feature" "-sramecc" "-target-feature" "+xnack"
 // CHECK-TARGET-ID: clang-offload-packager{{.*}}arch=gfx90a:sramecc-:xnack+,kind=openmp,feature=-sramecc,feature=+xnack
 
 // RUN: not %clang -### -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a,gfx90a:xnack+ \
diff --git a/clang/test/Driver/baremetal-ld.c b/clang/test/Driver/baremetal-ld.c
new file mode 100644
index 0000000000000000000000000000000000000000..ec61b42b6f4875e6f75ca8c5e9707bb8e4582efc
--- /dev/null
+++ b/clang/test/Driver/baremetal-ld.c
@@ -0,0 +1,6 @@
+// RUN: %clang -### --target=armv7-unknown-none-eabi -mcpu=cortex-m4 --sysroot= -fuse-ld=ld %s 2>&1 | FileCheck --check-prefix=NOLTO %s
+// NOLTO: {{".*ld.*"}} {{.*}}
+// NOLTO-NOT: "-plugin-opt=mcpu"
+
+// RUN: %clang -### --target=armv7-unknown-none-eabi -mcpu=cortex-m4 --sysroot= -fuse-ld=ld -flto -O3 %s 2>&1 | FileCheck --check-prefix=LTO %s
+// LTO: {{".*ld.*"}} {{.*}} "-plugin-opt=mcpu=cortex-m4" "-plugin-opt=O3"
diff --git a/clang/test/Driver/gcc-param.c b/clang/test/Driver/gcc-param.c
index 4672e1156ce7bb76682a54433c852525e3967810..7ae084784bf91d5867907b9ea7af05c70ebe3cd4 100644
--- a/clang/test/Driver/gcc-param.c
+++ b/clang/test/Driver/gcc-param.c
@@ -1,2 +1,2 @@
 // RUN: touch %t.o
-// RUN: %clang -Werror --param ssp-buffer-size=1 %t.o -###
+// RUN: %clang -Werror -Wno-msvc-not-found --param ssp-buffer-size=1 %t.o -###
diff --git a/clang/test/Driver/loongarch-default-features.c b/clang/test/Driver/loongarch-features.c
similarity index 100%
rename from clang/test/Driver/loongarch-default-features.c
rename to clang/test/Driver/loongarch-features.c
diff --git a/clang/test/Interpreter/pretty-print.c b/clang/test/Interpreter/pretty-print.c
new file mode 100644
index 0000000000000000000000000000000000000000..d21749a649e1c0631a9e91492df66c72ce373ed1
--- /dev/null
+++ b/clang/test/Interpreter/pretty-print.c
@@ -0,0 +1,11 @@
+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl -Xcc -xc  | FileCheck %s
+// RUN: cat %s | clang-repl -Xcc -std=c++11 | FileCheck %s
+
+// Fails with `Symbols not found: [ __clang_Interpreter_SetValueNoAlloc ]`.
+// UNSUPPORTED: hwasan
+
+const char* c_str = "Hello, world!"; c_str
+
+// CHECK: Not implement yet.
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index bad1374b9397ca9fffd4f601ab9fa1fb01bb87bb..2439025609b9f9ca6b4975b0c579bab667064e0b 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -1,15 +1,15 @@
 // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output.
 // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM
 // ARM: error: unknown target CPU 'not-a-cpu'
-// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
+// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, cortex-r52plus, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64
 // AARCH64: error: unknown target CPU 'not-a-cpu'
-// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, oryon-1, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_AARCH64
 // TUNE_AARCH64: error: unknown target CPU 'not-a-cpu'
-// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, oryon-1, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple i386--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86
 // X86: error: unknown target CPU 'not-a-cpu'
@@ -29,7 +29,7 @@
 
 // RUN: not %clang_cc1 -triple nvptx--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix NVPTX
 // NVPTX: error: unknown target CPU 'not-a-cpu'
-// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx9-generic, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx10-1-generic, gfx1010, gfx1011, gfx1012, gfx1013, gfx10-3-generic, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx11-generic, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx12-generic, gfx1200, gfx1201{{$}}
+// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx9-generic, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx10-1-generic, gfx1010, gfx1011, gfx1012, gfx1013, gfx10-3-generic, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx11-generic, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx12-generic, gfx1200, gfx1201{{$}}
 
 // RUN: not %clang_cc1 -triple r600--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix R600
 // R600: error: unknown target CPU 'not-a-cpu'
@@ -37,7 +37,7 @@
 
 // RUN: not %clang_cc1 -triple amdgcn--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AMDGCN
 // AMDGCN: error: unknown target CPU 'not-a-cpu'
-// AMDGCN-NEXT: note: valid target CPU values are: gfx600, tahiti, gfx601, pitcairn, verde, gfx602, hainan, oland, gfx700, kaveri, gfx701, hawaii, gfx702, gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo, gfx802, iceland, tonga, gfx803, fiji, polaris10, polaris11, gfx805, tongapro, gfx810, stoney, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201, gfx9-generic, gfx10-1-generic, gfx10-3-generic, gfx11-generic, gfx12-generic{{$}}
+// AMDGCN-NEXT: note: valid target CPU values are: gfx600, tahiti, gfx601, pitcairn, verde, gfx602, hainan, oland, gfx700, kaveri, gfx701, hawaii, gfx702, gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo, gfx802, iceland, tonga, gfx803, fiji, polaris10, polaris11, gfx805, tongapro, gfx810, stoney, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1200, gfx1201, gfx9-generic, gfx10-1-generic, gfx10-3-generic, gfx11-generic, gfx12-generic{{$}}
 
 // RUN: not %clang_cc1 -triple wasm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix WEBASM
 // WEBASM: error: unknown target CPU 'not-a-cpu'
diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm
new file mode 100644
index 0000000000000000000000000000000000000000..42ac061bc90b3f188a129025b97fb2d74ad5ee8b
--- /dev/null
+++ b/clang/test/Modules/no-transitive-decls-change.cppm
@@ -0,0 +1,112 @@
+// Testing that changing a declaration in an unused module file won't change 
+// the BMI of the current module file.
+//
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \
+// RUN:     %t/m-partA.v1.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \
+// RUN:     -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \
+// RUN:     -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm
+//
+// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \
+// RUN:     -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \
+// RUN:     -fmodule-file=m:partB=%t/m-partB.pcm
+// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \
+// RUN:     -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \
+// RUN:     -fmodule-file=m:partB=%t/m-partB.pcm
+// Since useBOnly only uses partB from module M, the change in partA shouldn't affect
+// useBOnly.
+// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null
+
+//--- m-partA.cppm
+export module m:partA;
+
+namespace A_Impl {
+    inline int getAImpl() {
+        return 43;
+    }
+
+    inline int getA2Impl() {
+        return 43;
+    }
+}
+
+namespace A {
+    using A_Impl::getAImpl;
+}
+
+export inline int getA() {
+    return 43;
+}
+
+export inline int getA2(int) {
+    return 88;
+}
+
+//--- m-partA.v1.cppm
+export module m:partA;
+
+namespace A_Impl {
+    inline int getAImpl() {
+        return 43;
+    }
+
+    inline int getA2Impl() {
+        return 43;
+    }
+}
+
+namespace A {
+    using A_Impl::getAImpl;
+    // Adding a new declaration without introducing a new declaration name.
+    using A_Impl::getA2Impl;
+}
+
+inline int getA() {
+    return 43;
+}
+
+inline int getA2(int) {
+    return 88;
+}
+
+// Now we add a new declaration without introducing new identifier and new types.
+// The consuming module which didn't use m:partA completely is expected to be
+// not changed.
+inline int getA(int) {
+    return 88;
+}
+
+//--- m-partB.cppm
+export module m:partB;
+
+export inline int getB() {
+    return 430;
+}
+
+//--- m.cppm
+export module m;
+export import :partA;
+export import :partB;
+
+//--- useBOnly.cppm
+export module useBOnly;
+import m;
+
+export inline int get() {
+    return getB();
+}
+
+//--- useAOnly.cppm
+export module useAOnly;
+import m;
+
+export inline int get() {
+    A<int> a;
+    return a.getValue();
+}
diff --git a/clang/test/OpenMP/distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/distribute_firstprivate_codegen.cpp
index 567d12119db2187842c67a6a9782230825918aad..6a15cae307623153f29bacc117ed8d6bfcec98cd 100644
--- a/clang/test/OpenMP/distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_firstprivate_codegen.cpp
@@ -493,9 +493,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -798,9 +797,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1158,9 +1156,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1461,9 +1458,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_lastprivate_codegen.cpp b/clang/test/OpenMP/distribute_lastprivate_codegen.cpp
index 5d1a12d27145e03786acae6a0b5a0787d6ad374f..510c5d4d88efc24740581d4439ecc6936e60e940 100644
--- a/clang/test/OpenMP/distribute_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_lastprivate_codegen.cpp
@@ -478,9 +478,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -801,9 +800,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1180,9 +1178,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1501,9 +1498,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_firstprivate_codegen.cpp
index faae599bb32afe7adc1f015792b3e4be42275035..11b18e871551eb6a9c31c83f2bcfb1ec9e365c65 100644
--- a/clang/test/OpenMP/distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_firstprivate_codegen.cpp
@@ -776,9 +776,8 @@ int main() {
 // CHECK8-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK8-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK8-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK8-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK8-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK8-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1211,9 +1210,8 @@ int main() {
 // CHECK8-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK8-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK8-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK8-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK8-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK8-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1695,9 +1693,8 @@ int main() {
 // CHECK10-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK10-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK10-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK10-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK10-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK10-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2124,9 +2121,8 @@ int main() {
 // CHECK10-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK10-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK10-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK10-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK10-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK10-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_lastprivate_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_lastprivate_codegen.cpp
index 18a86c5e5666399e8779fccaede3c0038f597693..29e1a35e90e0251194d93bd6d03f98cba659335f 100644
--- a/clang/test/OpenMP/distribute_parallel_for_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_lastprivate_codegen.cpp
@@ -752,9 +752,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1223,9 +1222,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1744,9 +1742,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2209,9 +2206,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_private_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_private_codegen.cpp
index 0969a0ca21e7ab56141c21f25661169dab8936d1..03c62ce699b3031a489d22dd515c9f247893d843 100644
--- a/clang/test/OpenMP/distribute_parallel_for_private_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_private_codegen.cpp
@@ -514,9 +514,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -835,9 +834,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1225,9 +1223,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1540,9 +1537,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_firstprivate_codegen.cpp
index 545ea9f0f43fdf947b726afd6942d0001886171e..cb2cbb039a4e01662ba6bf5538437cbcdad614cc 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_firstprivate_codegen.cpp
@@ -839,9 +839,8 @@ int main() {
 // CHECK8-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK8-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK8-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK8-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK8-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK8-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK8-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1288,9 +1287,8 @@ int main() {
 // CHECK8-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK8-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK8-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK8-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK8-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK8-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK8-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK8-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1786,9 +1784,8 @@ int main() {
 // CHECK10-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK10-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK10-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK10-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK10-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK10-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK10-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2229,9 +2226,8 @@ int main() {
 // CHECK10-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK10-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK10-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK10-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK10-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK10-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK10-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK10-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2721,9 +2717,8 @@ int main() {
 // CHECK12-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK12-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK12-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK12-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK12-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK12-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK12-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK12-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK12-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK12-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2828,9 +2823,8 @@ int main() {
 // CHECK12-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK12-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK12-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK12-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK12-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK12-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK12-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK12-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK12-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK12-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK12-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3026,9 +3020,8 @@ int main() {
 // CHECK14-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK14-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK14-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK14-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK14-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK14-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK14-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK14-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK14-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK14-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK14-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -3131,9 +3124,8 @@ int main() {
 // CHECK14-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK14-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK14-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK14-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK14-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK14-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK14-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK14-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK14-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK14-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK14-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_lastprivate_codegen.cpp
index 675f1313460e659e5822f7375ff1a32bcd28fe38..c8c230b40c50ca9ac8f6aec1caa5a584c7af00b2 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_lastprivate_codegen.cpp
@@ -819,9 +819,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1304,9 +1303,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1839,9 +1837,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2318,9 +2315,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2853,9 +2849,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3021,9 +3016,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3260,9 +3254,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -3426,9 +3419,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_private_codegen.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_private_codegen.cpp
index a74af2d7c3932044ea650144b94ea6cacaf2d981..f884babe7e2d90d88477fef6a6131a7ea7e5f76c 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_private_codegen.cpp
@@ -568,9 +568,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -903,9 +902,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1307,9 +1305,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1636,9 +1633,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -2043,9 +2039,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2182,9 +2177,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2393,9 +2387,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -2530,9 +2523,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/distribute_private_codegen.cpp b/clang/test/OpenMP/distribute_private_codegen.cpp
index b6e796fb8027c05168fb1fdefce7e3fd79ce5410..7544d429934241c331c2adcbe745b5782c6c0134 100644
--- a/clang/test/OpenMP/distribute_private_codegen.cpp
+++ b/clang/test/OpenMP/distribute_private_codegen.cpp
@@ -344,9 +344,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -661,9 +660,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -950,9 +948,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1265,9 +1262,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/distribute_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/distribute_simd_firstprivate_codegen.cpp
index 1b5950ce11d7fa2af7edcdfdde6fd55213266424..35284a50c3866672bf81270ac288d430ef4b21ca 100644
--- a/clang/test/OpenMP/distribute_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_simd_firstprivate_codegen.cpp
@@ -546,9 +546,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -858,9 +857,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1225,9 +1223,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1535,9 +1532,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1900,9 +1896,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2007,9 +2002,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2205,9 +2199,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2310,9 +2303,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/distribute_simd_lastprivate_codegen.cpp
index 92b73b220bcc218e30c0e86b180df8b4aa1636b1..33c488eedf3a9c51ab27c91e98cd7ee3627e5025 100644
--- a/clang/test/OpenMP/distribute_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/distribute_simd_lastprivate_codegen.cpp
@@ -533,9 +533,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -863,9 +862,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1249,9 +1247,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1577,9 +1574,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1967,9 +1963,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2135,9 +2130,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2374,9 +2368,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2540,9 +2533,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/distribute_simd_private_codegen.cpp b/clang/test/OpenMP/distribute_simd_private_codegen.cpp
index 93b2bd8b12a8ee3b738b7b950d7bf4c3c9a08799..534903e0afbe77ea5c824402d06fef8610372122 100644
--- a/clang/test/OpenMP/distribute_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/distribute_simd_private_codegen.cpp
@@ -389,9 +389,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -735,9 +734,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1035,9 +1033,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1379,9 +1376,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1685,9 +1681,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1849,9 +1844,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2066,9 +2060,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -2228,9 +2221,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/error_unsupport_feature.c b/clang/test/OpenMP/error_unsupport_feature.c
new file mode 100644
index 0000000000000000000000000000000000000000..611a8b4639c44e20f28c2dba6ef47760fb72f62e
--- /dev/null
+++ b/clang/test/OpenMP/error_unsupport_feature.c
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -emit-llvm -verify -fopenmp %s
+
+int main () {
+  int r = 0;
+#pragma omp scope reduction(+:r) // expected-error {{cannot compile this scope with FE outlining yet}}
+  r++;
+  return r;
+}
diff --git a/clang/test/OpenMP/for_firstprivate_codegen.cpp b/clang/test/OpenMP/for_firstprivate_codegen.cpp
index 79f76bd0f4482a149bd1f9f4822051b46e0fba2e..6f51b82668be3bafde507aa578c77b123c0f8333 100644
--- a/clang/test/OpenMP/for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_firstprivate_codegen.cpp
@@ -423,9 +423,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
diff --git a/clang/test/OpenMP/for_lastprivate_codegen.cpp b/clang/test/OpenMP/for_lastprivate_codegen.cpp
index c7ef60afcfc03293d695400a3d600127cafa86c6..f89969d85a3204764e95aa94274fad76ab98374c 100644
--- a/clang/test/OpenMP/for_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_lastprivate_codegen.cpp
@@ -374,9 +374,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 5, ptr @main.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]], ptr @_ZZ4mainE5sivar)
@@ -848,9 +847,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 128
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 128
@@ -2798,9 +2796,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK5-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 5, ptr @main.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]], ptr @_ZZ4mainE5sivar)
@@ -3290,9 +3287,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 128
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 128
diff --git a/clang/test/OpenMP/for_private_codegen.cpp b/clang/test/OpenMP/for_private_codegen.cpp
index 463e5305e9fb667cfa6f60531d7da7a72b54370f..d140f0f51cdf1e8264cbe812b6fff24c3e209199 100644
--- a/clang/test/OpenMP/for_private_codegen.cpp
+++ b/clang/test/OpenMP/for_private_codegen.cpp
@@ -121,9 +121,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 0, ptr @main.omp_outlined)
@@ -360,9 +359,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 0, ptr @_Z5tmainIiET_v.omp_outlined)
diff --git a/clang/test/OpenMP/for_reduction_codegen.cpp b/clang/test/OpenMP/for_reduction_codegen.cpp
index d025c6d6e571f88774cb764073ad4cb2bbe71d96..498845d1f4d15c1634c24a83e851968eb71c8604 100644
--- a/clang/test/OpenMP/for_reduction_codegen.cpp
+++ b/clang/test/OpenMP/for_reduction_codegen.cpp
@@ -556,13 +556,12 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [4 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_ELEMENT]], i64 1
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 2
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT1]], float noundef 3.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_ELEMENT1]], i64 1
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 3
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT2]], float noundef 4.000000e+00)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
@@ -3377,9 +3376,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiLi42EET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
diff --git a/clang/test/OpenMP/for_reduction_codegen_UDR.cpp b/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
index 999e062bbe264d278b84ba546ccdd2e6f2b4cb09..f0ab531e1d9924c529694cdd7dfefc5213e1daab 100644
--- a/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
+++ b/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
@@ -622,13 +622,12 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[TEST]])
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [4 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_ELEMENT]], i64 1
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 2
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT1]], float noundef 3.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_ELEMENT1]], i64 1
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 3
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT2]], float noundef 4.000000e+00)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[VAR1]])
@@ -2421,9 +2420,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiLi42EET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[VAR1]])
@@ -3318,13 +3316,12 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[TEST]])
 // CHECK3-NEXT:    store float 0.000000e+00, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [4 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK3-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK3-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK3-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_ELEMENT]], i64 1
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 2
 // CHECK3-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT1]], float noundef 3.000000e+00)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_ELEMENT1]], i64 1
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT2:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 3
 // CHECK3-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT2]], float noundef 4.000000e+00)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK3-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[VAR1]])
@@ -3584,9 +3581,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiLi42EET_v.vec, i64 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(12) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(12) [[VAR1]])
diff --git a/clang/test/OpenMP/parallel_copyin_codegen.cpp b/clang/test/OpenMP/parallel_copyin_codegen.cpp
index 57a563d49b31e89c6768dab35d4bd08c3c099e08..e653a7734161b724bba7fcffa85bc22194598a5f 100644
--- a/clang/test/OpenMP/parallel_copyin_codegen.cpp
+++ b/clang/test/OpenMP/parallel_copyin_codegen.cpp
@@ -297,9 +297,8 @@ void foo() {
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[TMP1]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[TMP1]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    ret ptr [[TMP2]]
@@ -574,9 +573,8 @@ void foo() {
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[TMP1]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0:%.*]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0:%.*]], ptr [[TMP1]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    ret ptr [[TMP2]]
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index e5374c384d2ab2423ffa22134e1a03fa8a026f88..1cfba8c3159584ee37b319b44d57a0a2cc590d89 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -258,9 +258,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], float 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[VAR]], float 3.000000e+00)
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -473,9 +472,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK1-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], i32 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[VAR]], i32 3)
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
@@ -1440,9 +1438,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], float 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float 2.000000e+00)
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[VAR]], float 3.000000e+00)
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -1655,9 +1652,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], i32 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 2)
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[VAR]], i32 3)
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
diff --git a/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
index fa631854ccae8870812f9c007ce1685d7b806d4c..bfb31c3003c3beb6a098887b9fe799f1cb91cc47 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
@@ -206,9 +206,8 @@ void array_func(int n, float a[n], St s[2]) {
 // CHECK-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // CHECK-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -569,9 +568,8 @@ void array_func(int n, float a[n], St s[2]) {
 // CHECK-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // CHECK-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
@@ -1519,9 +1517,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // SIMD-ONLY0-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[I]], align 4
@@ -1617,9 +1614,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // SIMD-ONLY0-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[I]], align 4
@@ -1842,9 +1838,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // SIMD-ONLY1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[I]], align 4
@@ -1940,9 +1935,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // SIMD-ONLY1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[I]], align 4
diff --git a/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
index 409a983f3121d9400d9a39a0b30442840a04cc88..f86d53777bd4a188c7f33874a3645b3d446024e2 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
@@ -212,9 +212,8 @@ void loop() {
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ev(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 4, ptr @main.omp_outlined, ptr [[VEC]], ptr [[T_VAR]], ptr [[S_ARR]], ptr [[VAR]])
@@ -580,9 +579,8 @@ void loop() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[VEC]], ptr [[T_VAR]], ptr [[S_ARR]], ptr [[VAR]])
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
index 46c35833a4e635c23bebf8a09cb61e2b43024235..68a7257a019c28c302979b54a1aec614dcf7ba01 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
@@ -206,9 +206,8 @@ void array_func(int n, float a[n], St s[2]) {
 // CHECK-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // CHECK-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // CHECK-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -569,9 +568,8 @@ void array_func(int n, float a[n], St s[2]) {
 // CHECK-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // CHECK-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
@@ -1523,9 +1521,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // SIMD-ONLY0-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // SIMD-ONLY0-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
@@ -1638,9 +1635,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // SIMD-ONLY0-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // SIMD-ONLY0-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// SIMD-ONLY0-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // SIMD-ONLY0-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // SIMD-ONLY0-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
@@ -1880,9 +1876,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1ERKS0_d(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]], ptr noundef nonnull align 8 dereferenceable(8) [[TTT]], double noundef 0.000000e+00)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // SIMD-ONLY1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // SIMD-ONLY1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
@@ -1995,9 +1990,8 @@ void array_func(int n, float a[n], St s[2]) {
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1ERKS0_i(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]], ptr noundef nonnull align 4 dereferenceable(4) [[TTT]], i32 noundef 0)
 // SIMD-ONLY1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // SIMD-ONLY1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// SIMD-ONLY1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // SIMD-ONLY1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // SIMD-ONLY1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
index 5e2c192497b5b93af8be9ab1097d02522a4bd5e9..8495cfa2e027e55828f31509c90778c8acc542b2 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
@@ -212,9 +212,8 @@ void loop() {
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ev(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 4, ptr @main.omp_outlined, ptr [[VEC]], ptr [[T_VAR]], ptr [[S_ARR]], ptr [[VAR]])
@@ -580,9 +579,8 @@ void loop() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[VEC]], ptr [[T_VAR]], ptr [[S_ARR]], ptr [[VAR]])
@@ -1755,9 +1753,8 @@ void loop() {
 // CHECK7-NEXT:    call void @_ZN1SIdEC1Ev(ptr noundef nonnull align 8 dereferenceable(8) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK7-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_BEGIN]], double noundef 1.000000e+00)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK7-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[S_ARR]], double noundef 1.000000e+00)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK7-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT]], double noundef 2.000000e+00)
 // CHECK7-NEXT:    call void @_ZN1SIdEC1Ed(ptr noundef nonnull align 8 dereferenceable(8) [[VAR]], double noundef 3.000000e+00)
 // CHECK7-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
@@ -1911,9 +1908,8 @@ void loop() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK7-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
diff --git a/clang/test/OpenMP/parallel_private_codegen.cpp b/clang/test/OpenMP/parallel_private_codegen.cpp
index 586399ba89a6bfaec83aed0d4c42ead5fdc5ae40..b86f4e48a33527750764654861db84303e65ff1e 100644
--- a/clang/test/OpenMP/parallel_private_codegen.cpp
+++ b/clang/test/OpenMP/parallel_private_codegen.cpp
@@ -182,9 +182,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 0, ptr @main.omp_outlined)
@@ -309,9 +308,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z5tmainIiET_v.omp_outlined)
diff --git a/clang/test/OpenMP/parallel_reduction_codegen.cpp b/clang/test/OpenMP/parallel_reduction_codegen.cpp
index 88a10a16615d0a04485366c59180da5fd6c7d496..d27ef4913a38649535151add7e460c77076cd8fa 100644
--- a/clang/test/OpenMP/parallel_reduction_codegen.cpp
+++ b/clang/test/OpenMP/parallel_reduction_codegen.cpp
@@ -511,9 +511,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
@@ -997,9 +996,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
diff --git a/clang/test/OpenMP/sections_firstprivate_codegen.cpp b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
index af67c429ac1125d89a9d5c9b30ccad39c1b216ba..3cf6cc29d56ae6241384c03ff1f8669b000053ce 100644
--- a/clang/test/OpenMP/sections_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
@@ -402,9 +402,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]])
diff --git a/clang/test/OpenMP/sections_lastprivate_codegen.cpp b/clang/test/OpenMP/sections_lastprivate_codegen.cpp
index fb7f29077bb50c655eecfa63cd7f99f5b8e807f3..126a3ba87e8182aa89ad13a8c6f50d9dbdeac9fa 100644
--- a/clang/test/OpenMP/sections_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_lastprivate_codegen.cpp
@@ -201,9 +201,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 5, ptr @main.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]], ptr @_ZZ4mainE5sivar)
@@ -465,9 +464,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]])
@@ -879,9 +877,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK5-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 5, ptr @main.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]], ptr @_ZZ4mainE5sivar)
@@ -1161,9 +1158,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK5-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]])
diff --git a/clang/test/OpenMP/sections_private_codegen.cpp b/clang/test/OpenMP/sections_private_codegen.cpp
index 034a8b38dac4c9d510f64308159d33fe761897b6..ee29fd953f97aee066e8f97dbd4cdbc78c4783a7 100644
--- a/clang/test/OpenMP/sections_private_codegen.cpp
+++ b/clang/test/OpenMP/sections_private_codegen.cpp
@@ -117,9 +117,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3:[0-9]+]], i32 0, ptr @main.omp_outlined)
@@ -274,9 +273,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 0, ptr @_Z5tmainIiET_v.omp_outlined)
diff --git a/clang/test/OpenMP/sections_reduction_codegen.cpp b/clang/test/OpenMP/sections_reduction_codegen.cpp
index 6166b5b3f9486c8ceb3cae5bbbde75841c6f831f..3eb7a1f8c54e5a87dcfe1c8e5f3850c969cc45f2 100644
--- a/clang/test/OpenMP/sections_reduction_codegen.cpp
+++ b/clang/test/OpenMP/sections_reduction_codegen.cpp
@@ -196,9 +196,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
@@ -546,9 +545,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR1]])
diff --git a/clang/test/OpenMP/simd_private_taskloop_codegen.cpp b/clang/test/OpenMP/simd_private_taskloop_codegen.cpp
index 50726b201523a0f78318ac98e5629fce5eb2031d..7e81f6f4dac600d271bf2713acd1235e12e672cf 100644
--- a/clang/test/OpenMP/simd_private_taskloop_codegen.cpp
+++ b/clang/test/OpenMP/simd_private_taskloop_codegen.cpp
@@ -470,9 +470,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    call void @__kmpc_taskgroup(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -850,9 +849,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.2], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_2]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_2]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    call void @__kmpc_taskgroup(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1209,9 +1207,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    call void @__kmpc_taskgroup(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1587,9 +1584,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.2], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_2]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_2]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    call void @__kmpc_taskgroup(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1955,9 +1951,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2123,9 +2118,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2342,9 +2336,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -2508,9 +2501,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/single_firstprivate_codegen.cpp b/clang/test/OpenMP/single_firstprivate_codegen.cpp
index f7156bf1c54d67c46e48c3ec3dc7ddd39cd1aa97..4b4c96655fccbff0932854284da1dfcc6a410cbd 100644
--- a/clang/test/OpenMP/single_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/single_firstprivate_codegen.cpp
@@ -357,9 +357,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], i32 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[VAR]], i32 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[VAR]])
diff --git a/clang/test/OpenMP/single_private_codegen.cpp b/clang/test/OpenMP/single_private_codegen.cpp
index e657d7bcbe8881c34d64a112dfdbd4a22cae67cb..ce79cfd1bd2690c5459552571940a4087c34d200 100644
--- a/clang/test/OpenMP/single_private_codegen.cpp
+++ b/clang/test/OpenMP/single_private_codegen.cpp
@@ -104,9 +104,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK1-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 0, ptr @main.omp_outlined)
@@ -226,9 +225,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z5tmainIiET_v.omp_outlined)
diff --git a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
index da69e19a890efa72621f2a9e7e176f7d8e4ea551..ab0a35f791a3168eff0394768a0fe650551924d6 100644
--- a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
@@ -546,9 +546,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1346,9 +1345,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_lastprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_lastprivate_codegen.cpp
index e23435d13e9aecd37d41e59d4e8d8932065f4151..adbab5d8775f7f3669a0944a7e6cd41626dbf337 100644
--- a/clang/test/OpenMP/target_teams_distribute_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_lastprivate_codegen.cpp
@@ -475,9 +475,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -799,9 +798,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1176,9 +1174,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1498,9 +1495,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index bf1fc1d7cbae5f6b94645116ca0f9822f9cc16a1..0806a71f95bcc524ee1f1fb816b17f0b33abdac0 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -737,9 +737,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1793,9 +1792,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_codegen.cpp
index ae900d2dc53b9c1914c580f0b6b029fae008a871..6f14f5113ee274eb9fefa9cc6b82bf7bbe2dbc7f 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_codegen.cpp
@@ -736,9 +736,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1214,9 +1213,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1736,9 +1734,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2208,9 +2205,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
index 49e4681c051530b603ac7c12bc8859effa6a1488..05ab657a3c83ea3e1811ca597a076c18873e886a 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
@@ -564,9 +564,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1291,9 +1290,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
index ea05aecebf3713cbd319dcee8509eb4520fc279f..fbf0b842542258ba6c79b2da6191f5bc86e25f84 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
@@ -749,9 +749,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1833,9 +1832,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2856,9 +2854,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3177,9 +3174,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
index b2b9e2082fa3a8dd1e01d4de683805c26930587f..cb8e091e8b02cdf270df3e3764ec9d1b6d6a49a1 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
@@ -764,9 +764,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1256,9 +1255,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1792,9 +1790,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2278,9 +2275,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2848,9 +2844,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3012,9 +3007,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3247,9 +3241,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -3409,9 +3402,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
index 0cc07c86aaea7d7e694730c8a4ca53cd5536f296..c7b919cfbb443a0a913cd89fbf9afcf14539a9c3 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
@@ -578,9 +578,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1333,9 +1332,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -2178,9 +2176,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK7-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2554,9 +2551,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK9-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_private_codegen.cpp
index 3cb2d2c2f9b43ef8d18036ae718a1024be5984e6..b718173246f383fb1356bd5f64cab1f1c395e280 100644
--- a/clang/test/OpenMP/target_teams_distribute_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_private_codegen.cpp
@@ -402,9 +402,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -928,9 +927,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_codegen.cpp
index d22aeee120d5d5caf73e9e489cf0c56ab4f26fe9..3e1cfb00209041b22d8d29a6d5f3f38371ac90c1 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_codegen.cpp
@@ -553,9 +553,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1367,9 +1366,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1911,9 +1909,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2231,9 +2228,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_codegen.cpp
index 93c570ac0604ff4e3ca364a9188b297fc8a37f26..fc17caae823282d71e5a095e1614e4380bcbbfc4 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_codegen.cpp
@@ -525,9 +525,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -856,9 +855,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1240,9 +1238,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1569,9 +1566,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1955,9 +1951,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2118,9 +2113,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2352,9 +2346,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2513,9 +2506,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_private_codegen.cpp
index 4e9b4e1027e6863d619fddea390ed445c33028a7..c1f3530e484c27cc8b783c6466dab5a348b71d9e 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_private_codegen.cpp
@@ -409,9 +409,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -949,9 +948,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1377,9 +1375,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1753,9 +1750,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp b/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
index 9b3d77f5b0adcf1893cff7df0451c9bee3396feb..5efad5b4327cfc61f48c0534f351fc290866a460 100644
--- a/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
@@ -463,9 +463,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -989,9 +988,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
index 8b3e657428ec64253c51669fe91f4a88e90e40af..2ebedb8dd03eeece28a7640164113b57ae5ae4a2 100644
--- a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
@@ -549,9 +549,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1351,9 +1350,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_lastprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_lastprivate_codegen.cpp
index 66c952f3281fb54e3f0454c3acef1f933df2b0c3..96ba444d402efc485bfa8dd6665b6fb2d1b81bb2 100644
--- a/clang/test/OpenMP/teams_distribute_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_lastprivate_codegen.cpp
@@ -469,9 +469,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -789,9 +788,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1165,9 +1163,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1483,9 +1480,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
index 0726fb659e7d15d990cf2157d43efd9ccf2e3ac6..8c411dee8a97adbb7611dbd9b06cc079fad4bdb7 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -713,9 +713,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1771,9 +1770,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_lastprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_lastprivate_codegen.cpp
index 06fc87d7da4c6c3c626293fa8fa5a735939b3b21..3e4afe969013e94b67b89e3b9fddb965c8e250ee 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_lastprivate_codegen.cpp
@@ -717,9 +717,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1185,9 +1184,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1703,9 +1701,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2165,9 +2162,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_private_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_private_codegen.cpp
index 4d10d16f47ed8716661415c52c45a663de983f5d..c9535d156d2ccb200797efc45d45d6a7e7048ad2 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_private_codegen.cpp
@@ -526,9 +526,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1253,9 +1252,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
index 7d9e2ab89676f689576d3084493a5d1aa54bd0c7..7c13f557c1c13ac8176b401a1c55cd7f8ee37d6e 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
@@ -730,9 +730,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1816,9 +1815,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2496,9 +2494,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2819,9 +2816,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
index 8643cb9bb84a8bd8ac3ba2c562cfbcfe6c7430c8..77a51286ff4970c2ba0716cd804b7717960887a6 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_lastprivate_codegen.cpp
@@ -790,9 +790,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1272,9 +1271,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1804,9 +1802,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2280,9 +2277,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2811,9 +2807,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2977,9 +2972,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -3214,9 +3208,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -3378,9 +3371,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_private_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_private_codegen.cpp
index f40acb42f9dae46b935242d75278471a760f1223..4d0c7f00d316a2ec17b41544260c1070e3a4869b 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_private_codegen.cpp
@@ -542,9 +542,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1297,9 +1296,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1832,9 +1830,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -2208,9 +2205,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_private_codegen.cpp b/clang/test/OpenMP/teams_distribute_private_codegen.cpp
index 78b42e3194c796067eeb3a7ce56a5005dacedb08..1f49d9334f31f703200c940357e701b156f82db8 100644
--- a/clang/test/OpenMP/teams_distribute_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_private_codegen.cpp
@@ -405,9 +405,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -931,9 +930,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_firstprivate_codegen.cpp
index 2d3fccd90c0a5d19867d7283a20bad0ce81ccdc6..9e6825d57cf9ad53ac8a4519f96b3b8bda591c1e 100644
--- a/clang/test/OpenMP/teams_distribute_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_firstprivate_codegen.cpp
@@ -556,9 +556,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1372,9 +1371,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1919,9 +1917,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2241,9 +2238,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_lastprivate_codegen.cpp
index ec95ab55b1552a9911931f9786a95418fa27d164..3f01eab5a7352bcc444df4761e84eb0ec9d416b7 100644
--- a/clang/test/OpenMP/teams_distribute_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_lastprivate_codegen.cpp
@@ -519,9 +519,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -846,9 +845,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK9-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -1229,9 +1227,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1554,9 +1551,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK11-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -1940,9 +1936,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2105,9 +2100,8 @@ int main() {
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK13-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK13-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK13-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK13-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK13-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK13-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
@@ -2341,9 +2335,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
@@ -2504,9 +2497,8 @@ int main() {
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK15-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK15-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK15-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK15-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK15-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK15-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
diff --git a/clang/test/OpenMP/teams_distribute_simd_private_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_private_codegen.cpp
index c839268ada753485516ae92981bf19e6c5410c83..4c87b2dcc111be8096ce95546c415324fe39d7b2 100644
--- a/clang/test/OpenMP/teams_distribute_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_private_codegen.cpp
@@ -413,9 +413,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -953,9 +952,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
@@ -1381,9 +1379,8 @@ int main() {
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK5-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK5-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK5-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK5-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK5-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK5-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK5-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -1757,9 +1754,8 @@ int main() {
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK7-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK7-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK7-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK7-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK7-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK7-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK7-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index 649fae99b21ac35a6a7ce3b38e325b010713ea52..304bd5dbb50ef387e8c73abd6e9d8c184d396cb1 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -283,9 +283,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], float 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float 2.000000e+00)
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[VAR]], float 3.000000e+00)
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -628,9 +627,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], i32 signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 signext 2)
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[VAR]], i32 signext 3)
 // CHECK9-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
@@ -1077,9 +1075,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], float 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float 2.000000e+00)
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr nonnull align 4 dereferenceable(4) [[VAR]], float 3.000000e+00)
 // CHECK11-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 4
@@ -1422,9 +1419,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ev(ptr nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[S_ARR]], i32 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 2)
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr nonnull align 4 dereferenceable(4) [[VAR]], i32 3)
 // CHECK11-NEXT:    [[TMP0:%.*]] = load i32, ptr [[T_VAR]], align 128
diff --git a/clang/test/OpenMP/teams_generic_loop_private_codegen.cpp b/clang/test/OpenMP/teams_generic_loop_private_codegen.cpp
index e955db129d1da60ffa94643609791298d910eecb..f44acf211b5e3bbc959cf6fcb941a641f162a2c6 100644
--- a/clang/test/OpenMP/teams_generic_loop_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_generic_loop_private_codegen.cpp
@@ -425,9 +425,8 @@ int main() {
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK1-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK1-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK1-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 8
 // CHECK1-NEXT:    store ptr undef, ptr [[_TMP1]], align 8
@@ -951,9 +950,8 @@ int main() {
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK3-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK3-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK3-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK3-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK3-NEXT:    store ptr [[TEST]], ptr [[VAR]], align 4
 // CHECK3-NEXT:    store ptr undef, ptr [[_TMP1]], align 4
diff --git a/clang/test/OpenMP/teams_private_codegen.cpp b/clang/test/OpenMP/teams_private_codegen.cpp
index 1d0b2435ff005819dd23f8d460f510c769c6f901..81c0ea77f1b048715ca51c96195c86791f86ab60 100644
--- a/clang/test/OpenMP/teams_private_codegen.cpp
+++ b/clang/test/OpenMP/teams_private_codegen.cpp
@@ -551,9 +551,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK9-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK9-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
@@ -718,9 +717,8 @@ int main() {
 // CHECK9-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK9-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK9-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i64 8, i1 false)
-// CHECK9-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i64 0, i64 0
-// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef signext 1)
-// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i64 1
+// CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef signext 1)
+// CHECK9-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
 // CHECK9-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef signext 3)
 // CHECK9-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
@@ -1141,9 +1139,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[TEST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 4
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC]], ptr align 4 @__const.main.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], float noundef 1.000000e+00)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], float noundef 1.000000e+00)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], float noundef 2.000000e+00)
 // CHECK11-NEXT:    call void @_ZN1SIfEC1Ef(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], float noundef 3.000000e+00)
 // CHECK11-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
@@ -1308,9 +1305,8 @@ int main() {
 // CHECK11-NEXT:    call void @_ZN3SSTIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[SST]])
 // CHECK11-NEXT:    store i32 0, ptr [[T_VAR]], align 128
 // CHECK11-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC]], ptr align 128 @__const._Z5tmainIiET_v.vec, i32 8, i1 false)
-// CHECK11-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
-// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN]], i32 noundef 1)
-// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYINIT_BEGIN]], i32 1
+// CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[S_ARR]], i32 noundef 1)
+// CHECK11-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
 // CHECK11-NEXT:    call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]], i32 noundef 3)
 // CHECK11-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
diff --git a/clang/test/OpenMP/threadprivate_codegen.cpp b/clang/test/OpenMP/threadprivate_codegen.cpp
index 5087451b944b981015659dd47ed114381a3093e8..2ee328008cc4bf09a143981efe966e9cdf483478 100644
--- a/clang/test/OpenMP/threadprivate_codegen.cpp
+++ b/clang/test/OpenMP/threadprivate_codegen.cpp
@@ -1028,46 +1028,43 @@ int foobar() {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[ARRAYINIT_ENDOFINIT:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[ARRAYINIT_ENDOFINIT2:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[ARRAYINIT_ENDOFINIT1:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[ARRAYINIT_ENDOFINIT9:%.*]] = alloca ptr, align 8
+// CHECK1-NEXT:    [[ARRAYINIT_ENDOFINIT7:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x [3 x %struct.S1]], ptr [[TMP1]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_BEGIN]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_BEGIN1]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN1]], i32 noundef 1)
+// CHECK1-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
+// CHECK1-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], i32 noundef 1)
 // CHECK1-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
 // CHECK1:       invoke.cont:
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[ARRAYINIT_BEGIN1]], i64 1
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[TMP1]], i64 1
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
 // CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
-// CHECK1-NEXT:            to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]]
-// CHECK1:       invoke.cont3:
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT4:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT]], i64 1
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT4]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT4]], i32 noundef 3)
-// CHECK1-NEXT:            to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD]]
-// CHECK1:       invoke.cont5:
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT7:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 1
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT7]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK1-NEXT:    [[ARRAYINIT_BEGIN8:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_ELEMENT7]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_BEGIN8]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN8]], i32 noundef 4)
-// CHECK1-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD10:%.*]]
+// CHECK1-NEXT:            to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]]
+// CHECK1:       invoke.cont2:
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT3:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[TMP1]], i64 2
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT3]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT3]], i32 noundef 3)
+// CHECK1-NEXT:            to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]]
+// CHECK1:       invoke.cont4:
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT6:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 1
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT6]], i32 noundef 4)
+// CHECK1-NEXT:            to label [[INVOKE_CONT9:%.*]] unwind label [[LPAD8:%.*]]
+// CHECK1:       invoke.cont9:
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 1
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT10]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT10]], i32 noundef 5)
+// CHECK1-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD8]]
 // CHECK1:       invoke.cont11:
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_BEGIN8]], i64 1
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 5)
-// CHECK1-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD10]]
+// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 2
+// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 6)
+// CHECK1-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD8]]
 // CHECK1:       invoke.cont13:
-// CHECK1-NEXT:    [[ARRAYINIT_ELEMENT14:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT12]], i64 1
-// CHECK1-NEXT:    store ptr [[ARRAYINIT_ELEMENT14]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT14]], i32 noundef 6)
-// CHECK1-NEXT:            to label [[INVOKE_CONT15:%.*]] unwind label [[LPAD10]]
-// CHECK1:       invoke.cont15:
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK1-NEXT:    ret ptr [[TMP2]]
 // CHECK1:       lpad:
@@ -1077,55 +1074,55 @@ int foobar() {
 // CHECK1-NEXT:    store ptr [[TMP4]], ptr [[EXN_SLOT]], align 8
 // CHECK1-NEXT:    [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 1
 // CHECK1-NEXT:    store i32 [[TMP5]], ptr [[EHSELECTOR_SLOT]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN1]], [[TMP6]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY:%.*]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[TMP1]], [[TMP6]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK1:       arraydestroy.body:
 // CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP6]], [[LPAD]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
 // CHECK1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
-// CHECK1-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAYINIT_BEGIN1]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6]], label [[ARRAYDESTROY_BODY]]
-// CHECK1:       arraydestroy.done6:
+// CHECK1-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[TMP1]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5]], label [[ARRAYDESTROY_BODY]]
+// CHECK1:       arraydestroy.done5:
 // CHECK1-NEXT:    br label [[EHCLEANUP:%.*]]
-// CHECK1:       lpad10:
+// CHECK1:       lpad8:
 // CHECK1-NEXT:    [[TMP7:%.*]] = landingpad { ptr, i32 }
 // CHECK1-NEXT:            cleanup
 // CHECK1-NEXT:    [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 0
 // CHECK1-NEXT:    store ptr [[TMP8]], ptr [[EXN_SLOT]], align 8
 // CHECK1-NEXT:    [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 1
 // CHECK1-NEXT:    store i32 [[TMP9]], ptr [[EHSELECTOR_SLOT]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY16:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN8]], [[TMP10]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY16]], label [[ARRAYDESTROY_DONE21:%.*]], label [[ARRAYDESTROY_BODY17:%.*]]
-// CHECK1:       arraydestroy.body17:
-// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST18:%.*]] = phi ptr [ [[TMP10]], [[LPAD10]] ], [ [[ARRAYDESTROY_ELEMENT19:%.*]], [[ARRAYDESTROY_BODY17]] ]
-// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENT19]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST18]], i64 -1
-// CHECK1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT19]]) #[[ATTR3]]
-// CHECK1-NEXT:    [[ARRAYDESTROY_DONE20:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT19]], [[ARRAYINIT_BEGIN8]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE20]], label [[ARRAYDESTROY_DONE21]], label [[ARRAYDESTROY_BODY17]]
-// CHECK1:       arraydestroy.done21:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY14:%.*]] = icmp eq ptr [[ARRAYINIT_ELEMENT6]], [[TMP10]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY14]], label [[ARRAYDESTROY_DONE19:%.*]], label [[ARRAYDESTROY_BODY15:%.*]]
+// CHECK1:       arraydestroy.body15:
+// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST16:%.*]] = phi ptr [ [[TMP10]], [[LPAD8]] ], [ [[ARRAYDESTROY_ELEMENT17:%.*]], [[ARRAYDESTROY_BODY15]] ]
+// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENT17]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST16]], i64 -1
+// CHECK1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT17]]) #[[ATTR3]]
+// CHECK1-NEXT:    [[ARRAYDESTROY_DONE18:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT17]], [[ARRAYINIT_ELEMENT6]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE18]], label [[ARRAYDESTROY_DONE19]], label [[ARRAYDESTROY_BODY15]]
+// CHECK1:       arraydestroy.done19:
 // CHECK1-NEXT:    br label [[EHCLEANUP]]
 // CHECK1:       ehcleanup:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK1-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0
+// CHECK1-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    [[PAD_ARRAYEND:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP11]], i64 0, i64 0
-// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY22:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY22]], label [[ARRAYDESTROY_DONE27:%.*]], label [[ARRAYDESTROY_BODY23:%.*]]
-// CHECK1:       arraydestroy.body23:
-// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST24:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT25:%.*]], [[ARRAYDESTROY_BODY23]] ]
-// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENT25]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST24]], i64 -1
-// CHECK1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT25]]) #[[ATTR3]]
-// CHECK1-NEXT:    [[ARRAYDESTROY_DONE26:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT25]], [[PAD_ARRAYBEGIN]]
-// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE26]], label [[ARRAYDESTROY_DONE27]], label [[ARRAYDESTROY_BODY23]]
-// CHECK1:       arraydestroy.done27:
+// CHECK1-NEXT:    [[ARRAYDESTROY_ISEMPTY20:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY20]], label [[ARRAYDESTROY_DONE25:%.*]], label [[ARRAYDESTROY_BODY21:%.*]]
+// CHECK1:       arraydestroy.body21:
+// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST22:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT23:%.*]], [[ARRAYDESTROY_BODY21]] ]
+// CHECK1-NEXT:    [[ARRAYDESTROY_ELEMENT23]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST22]], i64 -1
+// CHECK1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT23]]) #[[ATTR3]]
+// CHECK1-NEXT:    [[ARRAYDESTROY_DONE24:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT23]], [[PAD_ARRAYBEGIN]]
+// CHECK1-NEXT:    br i1 [[ARRAYDESTROY_DONE24]], label [[ARRAYDESTROY_DONE25]], label [[ARRAYDESTROY_BODY21]]
+// CHECK1:       arraydestroy.done25:
 // CHECK1-NEXT:    br label [[EH_RESUME:%.*]]
 // CHECK1:       eh.resume:
 // CHECK1-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
 // CHECK1-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
 // CHECK1-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } poison, ptr [[EXN]], 0
-// CHECK1-NEXT:    [[LPAD_VAL28:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
-// CHECK1-NEXT:    resume { ptr, i32 } [[LPAD_VAL28]]
+// CHECK1-NEXT:    [[LPAD_VAL26:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+// CHECK1-NEXT:    resume { ptr, i32 } [[LPAD_VAL26]]
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@.__kmpc_global_dtor_..2
@@ -1880,46 +1877,43 @@ int foobar() {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[ARRAYINIT_ENDOFINIT:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[ARRAYINIT_ENDOFINIT2:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[ARRAYINIT_ENDOFINIT1:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[ARRAYINIT_ENDOFINIT9:%.*]] = alloca ptr, align 8
+// CHECK2-NEXT:    [[ARRAYINIT_ENDOFINIT7:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK2-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x [3 x %struct.S1]], ptr [[TMP1]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_BEGIN]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK2-NEXT:    [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_BEGIN1]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN1]], i32 noundef 1)
+// CHECK2-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
+// CHECK2-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], i32 noundef 1)
 // CHECK2-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
 // CHECK2:       invoke.cont:
-// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[ARRAYINIT_BEGIN1]], i64 1
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
+// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[TMP1]], i64 1
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
 // CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
-// CHECK2-NEXT:            to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]]
-// CHECK2:       invoke.cont3:
-// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT4:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT]], i64 1
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT4]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT4]], i32 noundef 3)
-// CHECK2-NEXT:            to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD]]
-// CHECK2:       invoke.cont5:
-// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT7:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 1
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT7]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK2-NEXT:    [[ARRAYINIT_BEGIN8:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_ELEMENT7]], i64 0, i64 0
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_BEGIN8]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN8]], i32 noundef 4)
-// CHECK2-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD10:%.*]]
+// CHECK2-NEXT:            to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]]
+// CHECK2:       invoke.cont2:
+// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT3:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[TMP1]], i64 2
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT3]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT3]], i32 noundef 3)
+// CHECK2-NEXT:            to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]]
+// CHECK2:       invoke.cont4:
+// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT6:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 1
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT]], align 8
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT6]], i32 noundef 4)
+// CHECK2-NEXT:            to label [[INVOKE_CONT9:%.*]] unwind label [[LPAD8:%.*]]
+// CHECK2:       invoke.cont9:
+// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 1
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT10]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT10]], i32 noundef 5)
+// CHECK2-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD8]]
 // CHECK2:       invoke.cont11:
-// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_BEGIN8]], i64 1
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 5)
-// CHECK2-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD10]]
+// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 2
+// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 6)
+// CHECK2-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD8]]
 // CHECK2:       invoke.cont13:
-// CHECK2-NEXT:    [[ARRAYINIT_ELEMENT14:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT12]], i64 1
-// CHECK2-NEXT:    store ptr [[ARRAYINIT_ELEMENT14]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT14]], i32 noundef 6)
-// CHECK2-NEXT:            to label [[INVOKE_CONT15:%.*]] unwind label [[LPAD10]]
-// CHECK2:       invoke.cont15:
 // CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
 // CHECK2-NEXT:    ret ptr [[TMP2]]
 // CHECK2:       lpad:
@@ -1929,55 +1923,55 @@ int foobar() {
 // CHECK2-NEXT:    store ptr [[TMP4]], ptr [[EXN_SLOT]], align 8
 // CHECK2-NEXT:    [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 1
 // CHECK2-NEXT:    store i32 [[TMP5]], ptr [[EHSELECTOR_SLOT]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT2]], align 8
-// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN1]], [[TMP6]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY:%.*]]
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT1]], align 8
+// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[TMP1]], [[TMP6]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK2:       arraydestroy.body:
 // CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP6]], [[LPAD]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
 // CHECK2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
-// CHECK2-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAYINIT_BEGIN1]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6]], label [[ARRAYDESTROY_BODY]]
-// CHECK2:       arraydestroy.done6:
+// CHECK2-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[TMP1]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5]], label [[ARRAYDESTROY_BODY]]
+// CHECK2:       arraydestroy.done5:
 // CHECK2-NEXT:    br label [[EHCLEANUP:%.*]]
-// CHECK2:       lpad10:
+// CHECK2:       lpad8:
 // CHECK2-NEXT:    [[TMP7:%.*]] = landingpad { ptr, i32 }
 // CHECK2-NEXT:            cleanup
 // CHECK2-NEXT:    [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 0
 // CHECK2-NEXT:    store ptr [[TMP8]], ptr [[EXN_SLOT]], align 8
 // CHECK2-NEXT:    [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 1
 // CHECK2-NEXT:    store i32 [[TMP9]], ptr [[EHSELECTOR_SLOT]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT9]], align 8
-// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY16:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN8]], [[TMP10]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY16]], label [[ARRAYDESTROY_DONE21:%.*]], label [[ARRAYDESTROY_BODY17:%.*]]
-// CHECK2:       arraydestroy.body17:
-// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST18:%.*]] = phi ptr [ [[TMP10]], [[LPAD10]] ], [ [[ARRAYDESTROY_ELEMENT19:%.*]], [[ARRAYDESTROY_BODY17]] ]
-// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENT19]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST18]], i64 -1
-// CHECK2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT19]]) #[[ATTR3]]
-// CHECK2-NEXT:    [[ARRAYDESTROY_DONE20:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT19]], [[ARRAYINIT_BEGIN8]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE20]], label [[ARRAYDESTROY_DONE21]], label [[ARRAYDESTROY_BODY17]]
-// CHECK2:       arraydestroy.done21:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT7]], align 8
+// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY14:%.*]] = icmp eq ptr [[ARRAYINIT_ELEMENT6]], [[TMP10]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY14]], label [[ARRAYDESTROY_DONE19:%.*]], label [[ARRAYDESTROY_BODY15:%.*]]
+// CHECK2:       arraydestroy.body15:
+// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST16:%.*]] = phi ptr [ [[TMP10]], [[LPAD8]] ], [ [[ARRAYDESTROY_ELEMENT17:%.*]], [[ARRAYDESTROY_BODY15]] ]
+// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENT17]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST16]], i64 -1
+// CHECK2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT17]]) #[[ATTR3]]
+// CHECK2-NEXT:    [[ARRAYDESTROY_DONE18:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT17]], [[ARRAYINIT_ELEMENT6]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE18]], label [[ARRAYDESTROY_DONE19]], label [[ARRAYDESTROY_BODY15]]
+// CHECK2:       arraydestroy.done19:
 // CHECK2-NEXT:    br label [[EHCLEANUP]]
 // CHECK2:       ehcleanup:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT]], align 8
-// CHECK2-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0
+// CHECK2-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    [[PAD_ARRAYEND:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP11]], i64 0, i64 0
-// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY22:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY22]], label [[ARRAYDESTROY_DONE27:%.*]], label [[ARRAYDESTROY_BODY23:%.*]]
-// CHECK2:       arraydestroy.body23:
-// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST24:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT25:%.*]], [[ARRAYDESTROY_BODY23]] ]
-// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENT25]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST24]], i64 -1
-// CHECK2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT25]]) #[[ATTR3]]
-// CHECK2-NEXT:    [[ARRAYDESTROY_DONE26:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT25]], [[PAD_ARRAYBEGIN]]
-// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE26]], label [[ARRAYDESTROY_DONE27]], label [[ARRAYDESTROY_BODY23]]
-// CHECK2:       arraydestroy.done27:
+// CHECK2-NEXT:    [[ARRAYDESTROY_ISEMPTY20:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY20]], label [[ARRAYDESTROY_DONE25:%.*]], label [[ARRAYDESTROY_BODY21:%.*]]
+// CHECK2:       arraydestroy.body21:
+// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST22:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT23:%.*]], [[ARRAYDESTROY_BODY21]] ]
+// CHECK2-NEXT:    [[ARRAYDESTROY_ELEMENT23]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST22]], i64 -1
+// CHECK2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT23]]) #[[ATTR3]]
+// CHECK2-NEXT:    [[ARRAYDESTROY_DONE24:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT23]], [[PAD_ARRAYBEGIN]]
+// CHECK2-NEXT:    br i1 [[ARRAYDESTROY_DONE24]], label [[ARRAYDESTROY_DONE25]], label [[ARRAYDESTROY_BODY21]]
+// CHECK2:       arraydestroy.done25:
 // CHECK2-NEXT:    br label [[EH_RESUME:%.*]]
 // CHECK2:       eh.resume:
 // CHECK2-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
 // CHECK2-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
 // CHECK2-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } poison, ptr [[EXN]], 0
-// CHECK2-NEXT:    [[LPAD_VAL28:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
-// CHECK2-NEXT:    resume { ptr, i32 } [[LPAD_VAL28]]
+// CHECK2-NEXT:    [[LPAD_VAL26:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+// CHECK2-NEXT:    resume { ptr, i32 } [[LPAD_VAL26]]
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@.__kmpc_global_dtor_..4
@@ -6505,47 +6499,44 @@ int foobar() {
 // DEBUG1-NEXT:  entry:
 // DEBUG1-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // DEBUG1-NEXT:    [[ARRAYINIT_ENDOFINIT:%.*]] = alloca ptr, align 8
-// DEBUG1-NEXT:    [[ARRAYINIT_ENDOFINIT2:%.*]] = alloca ptr, align 8
+// DEBUG1-NEXT:    [[ARRAYINIT_ENDOFINIT1:%.*]] = alloca ptr, align 8
 // DEBUG1-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 // DEBUG1-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-// DEBUG1-NEXT:    [[ARRAYINIT_ENDOFINIT9:%.*]] = alloca ptr, align 8
+// DEBUG1-NEXT:    [[ARRAYINIT_ENDOFINIT7:%.*]] = alloca ptr, align 8
 // DEBUG1-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // DEBUG1-NEXT:    tail call void @llvm.dbg.declare(metadata ptr [[DOTADDR]], metadata [[META143:![0-9]+]], metadata !DIExpression()), !dbg [[DBG144:![0-9]+]]
 // DEBUG1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !dbg [[DBG145:![0-9]+]]
-// DEBUG1-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x [3 x %struct.S1]], ptr [[TMP1]], i64 0, i64 0, !dbg [[DBG146:![0-9]+]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_BEGIN]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0, !dbg [[DBG147:![0-9]+]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_BEGIN1]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG147]]
-// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN1]], i32 noundef 1)
+// DEBUG1-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG146:![0-9]+]]
+// DEBUG1-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG147:![0-9]+]]
+// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], i32 noundef 1)
 // DEBUG1-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]], !dbg [[DBG148:![0-9]+]]
 // DEBUG1:       invoke.cont:
-// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[ARRAYINIT_BEGIN1]], i64 1, !dbg [[DBG147]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG147]]
+// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[TMP1]], i64 1, !dbg [[DBG147]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG147]]
 // DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
-// DEBUG1-NEXT:            to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]], !dbg [[DBG149:![0-9]+]]
-// DEBUG1:       invoke.cont3:
-// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT4:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT]], i64 1, !dbg [[DBG147]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT4]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG147]]
-// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT4]], i32 noundef 3)
-// DEBUG1-NEXT:            to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD]], !dbg [[DBG150:![0-9]+]]
-// DEBUG1:       invoke.cont5:
-// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT7:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 1, !dbg [[DBG146]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT7]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[ARRAYINIT_BEGIN8:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_ELEMENT7]], i64 0, i64 0, !dbg [[DBG151:![0-9]+]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_BEGIN8]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG151]]
-// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN8]], i32 noundef 4)
-// DEBUG1-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD10:%.*]], !dbg [[DBG152:![0-9]+]]
+// DEBUG1-NEXT:            to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]], !dbg [[DBG149:![0-9]+]]
+// DEBUG1:       invoke.cont2:
+// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT3:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[TMP1]], i64 2, !dbg [[DBG147]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT3]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG147]]
+// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT3]], i32 noundef 3)
+// DEBUG1-NEXT:            to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]], !dbg [[DBG150:![0-9]+]]
+// DEBUG1:       invoke.cont4:
+// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT6:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 1, !dbg [[DBG146]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG146]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG151:![0-9]+]]
+// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT6]], i32 noundef 4)
+// DEBUG1-NEXT:            to label [[INVOKE_CONT9:%.*]] unwind label [[LPAD8:%.*]], !dbg [[DBG152:![0-9]+]]
+// DEBUG1:       invoke.cont9:
+// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 1, !dbg [[DBG151]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT10]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG151]]
+// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT10]], i32 noundef 5)
+// DEBUG1-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD8]], !dbg [[DBG153:![0-9]+]]
 // DEBUG1:       invoke.cont11:
-// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_BEGIN8]], i64 1, !dbg [[DBG151]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG151]]
-// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 5)
-// DEBUG1-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD10]], !dbg [[DBG153:![0-9]+]]
+// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 2, !dbg [[DBG151]]
+// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG151]]
+// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 6)
+// DEBUG1-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD8]], !dbg [[DBG154:![0-9]+]]
 // DEBUG1:       invoke.cont13:
-// DEBUG1-NEXT:    [[ARRAYINIT_ELEMENT14:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT12]], i64 1, !dbg [[DBG151]]
-// DEBUG1-NEXT:    store ptr [[ARRAYINIT_ELEMENT14]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG151]]
-// DEBUG1-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT14]], i32 noundef 6)
-// DEBUG1-NEXT:            to label [[INVOKE_CONT15:%.*]] unwind label [[LPAD10]], !dbg [[DBG154:![0-9]+]]
-// DEBUG1:       invoke.cont15:
 // DEBUG1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !dbg [[DBG145]]
 // DEBUG1-NEXT:    ret ptr [[TMP2]], !dbg [[DBG145]]
 // DEBUG1:       lpad:
@@ -6555,55 +6546,55 @@ int foobar() {
 // DEBUG1-NEXT:    store ptr [[TMP4]], ptr [[EXN_SLOT]], align 8, !dbg [[DBG144]]
 // DEBUG1-NEXT:    [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 1, !dbg [[DBG144]]
 // DEBUG1-NEXT:    store i32 [[TMP5]], ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG144]]
-// DEBUG1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG147]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN1]], [[TMP6]], !dbg [[DBG147]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY:%.*]], !dbg [[DBG147]]
+// DEBUG1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG147]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[TMP1]], [[TMP6]], !dbg [[DBG147]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY:%.*]], !dbg [[DBG147]]
 // DEBUG1:       arraydestroy.body:
 // DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP6]], [[LPAD]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ], !dbg [[DBG147]]
 // DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1, !dbg [[DBG147]]
 // DEBUG1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR4]], !dbg [[DBG147]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAYINIT_BEGIN1]], !dbg [[DBG147]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6]], label [[ARRAYDESTROY_BODY]], !dbg [[DBG147]]
-// DEBUG1:       arraydestroy.done6:
+// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[TMP1]], !dbg [[DBG147]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5]], label [[ARRAYDESTROY_BODY]], !dbg [[DBG147]]
+// DEBUG1:       arraydestroy.done5:
 // DEBUG1-NEXT:    br label [[EHCLEANUP:%.*]], !dbg [[DBG147]]
-// DEBUG1:       lpad10:
+// DEBUG1:       lpad8:
 // DEBUG1-NEXT:    [[TMP7:%.*]] = landingpad { ptr, i32 }
 // DEBUG1-NEXT:            cleanup, !dbg [[DBG144]]
 // DEBUG1-NEXT:    [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 0, !dbg [[DBG144]]
 // DEBUG1-NEXT:    store ptr [[TMP8]], ptr [[EXN_SLOT]], align 8, !dbg [[DBG144]]
 // DEBUG1-NEXT:    [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 1, !dbg [[DBG144]]
 // DEBUG1-NEXT:    store i32 [[TMP9]], ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG144]]
-// DEBUG1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG151]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY16:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN8]], [[TMP10]], !dbg [[DBG151]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY16]], label [[ARRAYDESTROY_DONE21:%.*]], label [[ARRAYDESTROY_BODY17:%.*]], !dbg [[DBG151]]
-// DEBUG1:       arraydestroy.body17:
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST18:%.*]] = phi ptr [ [[TMP10]], [[LPAD10]] ], [ [[ARRAYDESTROY_ELEMENT19:%.*]], [[ARRAYDESTROY_BODY17]] ], !dbg [[DBG151]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENT19]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST18]], i64 -1, !dbg [[DBG151]]
-// DEBUG1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT19]]) #[[ATTR4]], !dbg [[DBG151]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE20:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT19]], [[ARRAYINIT_BEGIN8]], !dbg [[DBG151]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE20]], label [[ARRAYDESTROY_DONE21]], label [[ARRAYDESTROY_BODY17]], !dbg [[DBG151]]
-// DEBUG1:       arraydestroy.done21:
+// DEBUG1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG151]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY14:%.*]] = icmp eq ptr [[ARRAYINIT_ELEMENT6]], [[TMP10]], !dbg [[DBG151]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY14]], label [[ARRAYDESTROY_DONE19:%.*]], label [[ARRAYDESTROY_BODY15:%.*]], !dbg [[DBG151]]
+// DEBUG1:       arraydestroy.body15:
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST16:%.*]] = phi ptr [ [[TMP10]], [[LPAD8]] ], [ [[ARRAYDESTROY_ELEMENT17:%.*]], [[ARRAYDESTROY_BODY15]] ], !dbg [[DBG151]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENT17]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST16]], i64 -1, !dbg [[DBG151]]
+// DEBUG1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT17]]) #[[ATTR4]], !dbg [[DBG151]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE18:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT17]], [[ARRAYINIT_ELEMENT6]], !dbg [[DBG151]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE18]], label [[ARRAYDESTROY_DONE19]], label [[ARRAYDESTROY_BODY15]], !dbg [[DBG151]]
+// DEBUG1:       arraydestroy.done19:
 // DEBUG1-NEXT:    br label [[EHCLEANUP]], !dbg [[DBG151]]
 // DEBUG1:       ehcleanup:
 // DEBUG1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0, !dbg [[DBG146]]
+// DEBUG1-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 0, i64 0, !dbg [[DBG146]]
 // DEBUG1-NEXT:    [[PAD_ARRAYEND:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP11]], i64 0, i64 0, !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY22:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]], !dbg [[DBG146]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY22]], label [[ARRAYDESTROY_DONE27:%.*]], label [[ARRAYDESTROY_BODY23:%.*]], !dbg [[DBG146]]
-// DEBUG1:       arraydestroy.body23:
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST24:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT25:%.*]], [[ARRAYDESTROY_BODY23]] ], !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENT25]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST24]], i64 -1, !dbg [[DBG146]]
-// DEBUG1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT25]]) #[[ATTR4]], !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE26:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT25]], [[PAD_ARRAYBEGIN]], !dbg [[DBG146]]
-// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE26]], label [[ARRAYDESTROY_DONE27]], label [[ARRAYDESTROY_BODY23]], !dbg [[DBG146]]
-// DEBUG1:       arraydestroy.done27:
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ISEMPTY20:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]], !dbg [[DBG146]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY20]], label [[ARRAYDESTROY_DONE25:%.*]], label [[ARRAYDESTROY_BODY21:%.*]], !dbg [[DBG146]]
+// DEBUG1:       arraydestroy.body21:
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENTPAST22:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT23:%.*]], [[ARRAYDESTROY_BODY21]] ], !dbg [[DBG146]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_ELEMENT23]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST22]], i64 -1, !dbg [[DBG146]]
+// DEBUG1-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT23]]) #[[ATTR4]], !dbg [[DBG146]]
+// DEBUG1-NEXT:    [[ARRAYDESTROY_DONE24:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT23]], [[PAD_ARRAYBEGIN]], !dbg [[DBG146]]
+// DEBUG1-NEXT:    br i1 [[ARRAYDESTROY_DONE24]], label [[ARRAYDESTROY_DONE25]], label [[ARRAYDESTROY_BODY21]], !dbg [[DBG146]]
+// DEBUG1:       arraydestroy.done25:
 // DEBUG1-NEXT:    br label [[EH_RESUME:%.*]], !dbg [[DBG146]]
 // DEBUG1:       eh.resume:
 // DEBUG1-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8, !dbg [[DBG146]]
 // DEBUG1-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG146]]
 // DEBUG1-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } poison, ptr [[EXN]], 0, !dbg [[DBG146]]
-// DEBUG1-NEXT:    [[LPAD_VAL28:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1, !dbg [[DBG146]]
-// DEBUG1-NEXT:    resume { ptr, i32 } [[LPAD_VAL28]], !dbg [[DBG146]]
+// DEBUG1-NEXT:    [[LPAD_VAL26:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1, !dbg [[DBG146]]
+// DEBUG1-NEXT:    resume { ptr, i32 } [[LPAD_VAL26]], !dbg [[DBG146]]
 //
 //
 // DEBUG1-LABEL: define {{[^@]+}}@.__kmpc_global_dtor_..2
@@ -7375,47 +7366,44 @@ int foobar() {
 // DEBUG2-NEXT:  entry:
 // DEBUG2-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
 // DEBUG2-NEXT:    [[ARRAYINIT_ENDOFINIT:%.*]] = alloca ptr, align 8
-// DEBUG2-NEXT:    [[ARRAYINIT_ENDOFINIT2:%.*]] = alloca ptr, align 8
+// DEBUG2-NEXT:    [[ARRAYINIT_ENDOFINIT1:%.*]] = alloca ptr, align 8
 // DEBUG2-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 // DEBUG2-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-// DEBUG2-NEXT:    [[ARRAYINIT_ENDOFINIT9:%.*]] = alloca ptr, align 8
+// DEBUG2-NEXT:    [[ARRAYINIT_ENDOFINIT7:%.*]] = alloca ptr, align 8
 // DEBUG2-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
 // DEBUG2-NEXT:    tail call void @llvm.dbg.declare(metadata ptr [[DOTADDR]], metadata [[META179:![0-9]+]], metadata !DIExpression()), !dbg [[DBG180:![0-9]+]]
 // DEBUG2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !dbg [[DBG181:![0-9]+]]
-// DEBUG2-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [2 x [3 x %struct.S1]], ptr [[TMP1]], i64 0, i64 0, !dbg [[DBG182:![0-9]+]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_BEGIN]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[ARRAYINIT_BEGIN1:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0, !dbg [[DBG183:![0-9]+]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_BEGIN1]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG183]]
-// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN1]], i32 noundef 1)
+// DEBUG2-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG182:![0-9]+]]
+// DEBUG2-NEXT:    store ptr [[TMP1]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG183:![0-9]+]]
+// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP1]], i32 noundef 1)
 // DEBUG2-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]], !dbg [[DBG184:![0-9]+]]
 // DEBUG2:       invoke.cont:
-// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[ARRAYINIT_BEGIN1]], i64 1, !dbg [[DBG183]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG183]]
+// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S1:%.*]], ptr [[TMP1]], i64 1, !dbg [[DBG183]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG183]]
 // DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
-// DEBUG2-NEXT:            to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]], !dbg [[DBG185:![0-9]+]]
-// DEBUG2:       invoke.cont3:
-// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT4:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT]], i64 1, !dbg [[DBG183]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT4]], ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG183]]
-// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT4]], i32 noundef 3)
-// DEBUG2-NEXT:            to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD]], !dbg [[DBG186:![0-9]+]]
-// DEBUG2:       invoke.cont5:
-// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT7:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 1, !dbg [[DBG182]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT7]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[ARRAYINIT_BEGIN8:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_ELEMENT7]], i64 0, i64 0, !dbg [[DBG187:![0-9]+]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_BEGIN8]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG187]]
-// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_BEGIN8]], i32 noundef 4)
-// DEBUG2-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD10:%.*]], !dbg [[DBG188:![0-9]+]]
+// DEBUG2-NEXT:            to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]], !dbg [[DBG185:![0-9]+]]
+// DEBUG2:       invoke.cont2:
+// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT3:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[TMP1]], i64 2, !dbg [[DBG183]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT3]], ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG183]]
+// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT3]], i32 noundef 3)
+// DEBUG2-NEXT:            to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]], !dbg [[DBG186:![0-9]+]]
+// DEBUG2:       invoke.cont4:
+// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT6:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 1, !dbg [[DBG182]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG182]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT6]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG187:![0-9]+]]
+// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT6]], i32 noundef 4)
+// DEBUG2-NEXT:            to label [[INVOKE_CONT9:%.*]] unwind label [[LPAD8:%.*]], !dbg [[DBG188:![0-9]+]]
+// DEBUG2:       invoke.cont9:
+// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 1, !dbg [[DBG187]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT10]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG187]]
+// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT10]], i32 noundef 5)
+// DEBUG2-NEXT:            to label [[INVOKE_CONT11:%.*]] unwind label [[LPAD8]], !dbg [[DBG189:![0-9]+]]
 // DEBUG2:       invoke.cont11:
-// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_BEGIN8]], i64 1, !dbg [[DBG187]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG187]]
-// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 5)
-// DEBUG2-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD10]], !dbg [[DBG189:![0-9]+]]
+// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT12:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT6]], i64 2, !dbg [[DBG187]]
+// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT12]], ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG187]]
+// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT12]], i32 noundef 6)
+// DEBUG2-NEXT:            to label [[INVOKE_CONT13:%.*]] unwind label [[LPAD8]], !dbg [[DBG190:![0-9]+]]
 // DEBUG2:       invoke.cont13:
-// DEBUG2-NEXT:    [[ARRAYINIT_ELEMENT14:%.*]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYINIT_ELEMENT12]], i64 1, !dbg [[DBG187]]
-// DEBUG2-NEXT:    store ptr [[ARRAYINIT_ELEMENT14]], ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG187]]
-// DEBUG2-NEXT:    invoke void @_ZN2S1C1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT14]], i32 noundef 6)
-// DEBUG2-NEXT:            to label [[INVOKE_CONT15:%.*]] unwind label [[LPAD10]], !dbg [[DBG190:![0-9]+]]
-// DEBUG2:       invoke.cont15:
 // DEBUG2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !dbg [[DBG181]]
 // DEBUG2-NEXT:    ret ptr [[TMP2]], !dbg [[DBG181]]
 // DEBUG2:       lpad:
@@ -7425,55 +7413,55 @@ int foobar() {
 // DEBUG2-NEXT:    store ptr [[TMP4]], ptr [[EXN_SLOT]], align 8, !dbg [[DBG180]]
 // DEBUG2-NEXT:    [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 1, !dbg [[DBG180]]
 // DEBUG2-NEXT:    store i32 [[TMP5]], ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG180]]
-// DEBUG2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT2]], align 8, !dbg [[DBG183]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN1]], [[TMP6]], !dbg [[DBG183]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY:%.*]], !dbg [[DBG183]]
+// DEBUG2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT1]], align 8, !dbg [[DBG183]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY:%.*]] = icmp eq ptr [[TMP1]], [[TMP6]], !dbg [[DBG183]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY:%.*]], !dbg [[DBG183]]
 // DEBUG2:       arraydestroy.body:
 // DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP6]], [[LPAD]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ], !dbg [[DBG183]]
 // DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1, !dbg [[DBG183]]
 // DEBUG2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR4]], !dbg [[DBG183]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAYINIT_BEGIN1]], !dbg [[DBG183]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6]], label [[ARRAYDESTROY_BODY]], !dbg [[DBG183]]
-// DEBUG2:       arraydestroy.done6:
+// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[TMP1]], !dbg [[DBG183]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5]], label [[ARRAYDESTROY_BODY]], !dbg [[DBG183]]
+// DEBUG2:       arraydestroy.done5:
 // DEBUG2-NEXT:    br label [[EHCLEANUP:%.*]], !dbg [[DBG183]]
-// DEBUG2:       lpad10:
+// DEBUG2:       lpad8:
 // DEBUG2-NEXT:    [[TMP7:%.*]] = landingpad { ptr, i32 }
 // DEBUG2-NEXT:            cleanup, !dbg [[DBG180]]
 // DEBUG2-NEXT:    [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 0, !dbg [[DBG180]]
 // DEBUG2-NEXT:    store ptr [[TMP8]], ptr [[EXN_SLOT]], align 8, !dbg [[DBG180]]
 // DEBUG2-NEXT:    [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 1, !dbg [[DBG180]]
 // DEBUG2-NEXT:    store i32 [[TMP9]], ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG180]]
-// DEBUG2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT9]], align 8, !dbg [[DBG187]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY16:%.*]] = icmp eq ptr [[ARRAYINIT_BEGIN8]], [[TMP10]], !dbg [[DBG187]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY16]], label [[ARRAYDESTROY_DONE21:%.*]], label [[ARRAYDESTROY_BODY17:%.*]], !dbg [[DBG187]]
-// DEBUG2:       arraydestroy.body17:
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST18:%.*]] = phi ptr [ [[TMP10]], [[LPAD10]] ], [ [[ARRAYDESTROY_ELEMENT19:%.*]], [[ARRAYDESTROY_BODY17]] ], !dbg [[DBG187]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENT19]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST18]], i64 -1, !dbg [[DBG187]]
-// DEBUG2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT19]]) #[[ATTR4]], !dbg [[DBG187]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE20:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT19]], [[ARRAYINIT_BEGIN8]], !dbg [[DBG187]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE20]], label [[ARRAYDESTROY_DONE21]], label [[ARRAYDESTROY_BODY17]], !dbg [[DBG187]]
-// DEBUG2:       arraydestroy.done21:
+// DEBUG2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT7]], align 8, !dbg [[DBG187]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY14:%.*]] = icmp eq ptr [[ARRAYINIT_ELEMENT6]], [[TMP10]], !dbg [[DBG187]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY14]], label [[ARRAYDESTROY_DONE19:%.*]], label [[ARRAYDESTROY_BODY15:%.*]], !dbg [[DBG187]]
+// DEBUG2:       arraydestroy.body15:
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST16:%.*]] = phi ptr [ [[TMP10]], [[LPAD8]] ], [ [[ARRAYDESTROY_ELEMENT17:%.*]], [[ARRAYDESTROY_BODY15]] ], !dbg [[DBG187]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENT17]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST16]], i64 -1, !dbg [[DBG187]]
+// DEBUG2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT17]]) #[[ATTR4]], !dbg [[DBG187]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE18:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT17]], [[ARRAYINIT_ELEMENT6]], !dbg [[DBG187]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE18]], label [[ARRAYDESTROY_DONE19]], label [[ARRAYDESTROY_BODY15]], !dbg [[DBG187]]
+// DEBUG2:       arraydestroy.done19:
 // DEBUG2-NEXT:    br label [[EHCLEANUP]], !dbg [[DBG187]]
 // DEBUG2:       ehcleanup:
 // DEBUG2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[ARRAYINIT_ENDOFINIT]], align 8, !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[ARRAYINIT_BEGIN]], i64 0, i64 0, !dbg [[DBG182]]
+// DEBUG2-NEXT:    [[PAD_ARRAYBEGIN:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP1]], i64 0, i64 0, !dbg [[DBG182]]
 // DEBUG2-NEXT:    [[PAD_ARRAYEND:%.*]] = getelementptr inbounds [3 x %struct.S1], ptr [[TMP11]], i64 0, i64 0, !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY22:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]], !dbg [[DBG182]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY22]], label [[ARRAYDESTROY_DONE27:%.*]], label [[ARRAYDESTROY_BODY23:%.*]], !dbg [[DBG182]]
-// DEBUG2:       arraydestroy.body23:
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST24:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT25:%.*]], [[ARRAYDESTROY_BODY23]] ], !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENT25]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST24]], i64 -1, !dbg [[DBG182]]
-// DEBUG2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT25]]) #[[ATTR4]], !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE26:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT25]], [[PAD_ARRAYBEGIN]], !dbg [[DBG182]]
-// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE26]], label [[ARRAYDESTROY_DONE27]], label [[ARRAYDESTROY_BODY23]], !dbg [[DBG182]]
-// DEBUG2:       arraydestroy.done27:
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ISEMPTY20:%.*]] = icmp eq ptr [[PAD_ARRAYBEGIN]], [[PAD_ARRAYEND]], !dbg [[DBG182]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_ISEMPTY20]], label [[ARRAYDESTROY_DONE25:%.*]], label [[ARRAYDESTROY_BODY21:%.*]], !dbg [[DBG182]]
+// DEBUG2:       arraydestroy.body21:
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENTPAST22:%.*]] = phi ptr [ [[PAD_ARRAYEND]], [[EHCLEANUP]] ], [ [[ARRAYDESTROY_ELEMENT23:%.*]], [[ARRAYDESTROY_BODY21]] ], !dbg [[DBG182]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_ELEMENT23]] = getelementptr inbounds [[STRUCT_S1]], ptr [[ARRAYDESTROY_ELEMENTPAST22]], i64 -1, !dbg [[DBG182]]
+// DEBUG2-NEXT:    call void @_ZN2S1D1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT23]]) #[[ATTR4]], !dbg [[DBG182]]
+// DEBUG2-NEXT:    [[ARRAYDESTROY_DONE24:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT23]], [[PAD_ARRAYBEGIN]], !dbg [[DBG182]]
+// DEBUG2-NEXT:    br i1 [[ARRAYDESTROY_DONE24]], label [[ARRAYDESTROY_DONE25]], label [[ARRAYDESTROY_BODY21]], !dbg [[DBG182]]
+// DEBUG2:       arraydestroy.done25:
 // DEBUG2-NEXT:    br label [[EH_RESUME:%.*]], !dbg [[DBG182]]
 // DEBUG2:       eh.resume:
 // DEBUG2-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8, !dbg [[DBG182]]
 // DEBUG2-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4, !dbg [[DBG182]]
 // DEBUG2-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } poison, ptr [[EXN]], 0, !dbg [[DBG182]]
-// DEBUG2-NEXT:    [[LPAD_VAL28:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1, !dbg [[DBG182]]
-// DEBUG2-NEXT:    resume { ptr, i32 } [[LPAD_VAL28]], !dbg [[DBG182]]
+// DEBUG2-NEXT:    [[LPAD_VAL26:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1, !dbg [[DBG182]]
+// DEBUG2-NEXT:    resume { ptr, i32 } [[LPAD_VAL26]], !dbg [[DBG182]]
 //
 //
 // DEBUG2-LABEL: define {{[^@]+}}@.__kmpc_global_dtor_..4
diff --git a/clang/test/PCH/cxx_paren_init.cpp b/clang/test/PCH/cxx_paren_init.cpp
index 9731ea7737c198308fec9d4feb86d9f2fedb02d6..298e3161007ce2127fab338ca33e6e265dabd90b 100644
--- a/clang/test/PCH/cxx_paren_init.cpp
+++ b/clang/test/PCH/cxx_paren_init.cpp
@@ -16,14 +16,13 @@ U u = baz(3);
 // CHECK-NEXT: [[ARR:%.*]] = alloca [4 x i32], align 16
 // CHECK-NEXT: store i32 [[A:%.*]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT: store i32 [[B:%.*]], ptr [[J_ADDR]], align 4
-// CHECK-NEXT: [[ARRINIT_BEGIN:%.*]] = getelementptr inbounds [4 x i32], ptr [[ARR]], i64 0, i64 0
 // CHECK-NEXT: [[TMP_0:%.*]] = load i32, ptr [[I_ADDR]], align 4
-// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARRINIT_BEGIN]], align 4
-// CHECK-NEXT: [[ARRINIT_ELEM:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 1
+// CHECK-NEXT: store i32 [[TMP_0]], ptr [[ARR]], align 4
+// CHECK-NEXT: [[ARRINIT_ELEM:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 1
 // CHECK-NEXT: [[TMP_1:%.*]] = load i32, ptr [[J_ADDR]], align 4
 // CHECK-NEXT: store i32 [[TMP_1]], ptr [[ARRINIT_ELEM]], align 4
-// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_ELEM]], i64 1
-// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[ARRINIT_BEGIN]], i64 4
+// CHECK-NEXT: [[ARRINIT_START:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 2
+// CHECK-NEXT: [[ARRINIT_END:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 4
 // CHECK-NEXT: br label [[ARRINIT_BODY:%.*]]
 // CHECK: [[ARRINIT_CUR:%.*]] = phi ptr [ [[ARRINIT_START]], %entry ], [ [[ARRINIT_NEXT:%.*]], [[ARRINIT_BODY]] ]
 // CHECK-NEXT: store i32 0, ptr [[ARRINIT_CUR]], align 4
diff --git a/clang/test/Preprocessor/hash_builtin.cpp b/clang/test/Preprocessor/hash_builtin.cpp
index 77d186c7883f22d14a5e831572da6a24ab0190ea..018b71eca418eb18f088c90d2e2d22cdfc8495aa 100644
--- a/clang/test/Preprocessor/hash_builtin.cpp
+++ b/clang/test/Preprocessor/hash_builtin.cpp
@@ -1,11 +1,14 @@
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx906 -E %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -E %s -o - | FileCheck %s --check-prefix=SPIRV-AMDGCN
 
 // CHECK: has_s_memtime_inst
+// SPIRV-AMDGCN: has_s_memtime_inst
 #if __has_builtin(__builtin_amdgcn_s_memtime)
   int has_s_memtime_inst;
 #endif
 
 // CHECK-NOT: has_gfx10_inst
+// SPIRV-AMDGCN: has_gfx10_inst
 #if __has_builtin(__builtin_amdgcn_mov_dpp8)
   int has_gfx10_inst;
 #endif
diff --git a/clang/test/Preprocessor/predefined-macros-no-warnings.c b/clang/test/Preprocessor/predefined-macros-no-warnings.c
index e0617f8de4da3856f6248d1dd6e847b72622b99d..722e3e77214b6474888e443cd5bc9ad1cf743f6d 100644
--- a/clang/test/Preprocessor/predefined-macros-no-warnings.c
+++ b/clang/test/Preprocessor/predefined-macros-no-warnings.c
@@ -173,6 +173,7 @@
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spir64
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv32
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv64
+// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv64-amd-amdhsa
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32-wasi
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32-emscripten
diff --git a/clang/test/Preprocessor/predefined-macros.c b/clang/test/Preprocessor/predefined-macros.c
index c4a9672f0814aad4c0e7506bb33008f7bec51eca..7f036bff401ca09506aae8be9b44c8a20f70c56f 100644
--- a/clang/test/Preprocessor/predefined-macros.c
+++ b/clang/test/Preprocessor/predefined-macros.c
@@ -236,6 +236,16 @@
 // CHECK-SPIRV64-DAG: #define __SPIRV64__ 1
 // CHECK-SPIRV64-NOT: #define __SPIRV32__ 1
 
+// RUN: %clang_cc1 %s -E -dM -o - -x cl -triple spirv64-amd-amdhsa \
+// RUN:   | FileCheck -match-full-lines %s --check-prefix=CHECK-SPIRV64-AMDGCN
+// CHECK-SPIRV64-AMDGCN-DAG: #define __IMAGE_SUPPORT__ 1
+// CHECK-SPIRV64-AMDGCN-DAG: #define __SPIRV__ 1
+// CHECK-SPIRV64-AMDGCN-DAG: #define __SPIRV64__ 1
+// CHECK-SPIRV64-AMDGCN-DAG: #define __AMD__ 1
+// CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGCN__ 1
+// CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGPU__ 1
+// CHECK-SPIRV64-AMDGCN-NOT: #define __SPIRV32__ 1
+
 // RUN: %clang_cc1 %s -E -dM -o - -x hip -triple x86_64-unknown-linux-gnu \
 // RUN:   | FileCheck -match-full-lines %s --check-prefix=CHECK-HIP
 // CHECK-HIP: #define __HIPCC__ 1
diff --git a/clang/test/Sema/aarch64-neon-target.c b/clang/test/Sema/aarch64-neon-target.c
index fa45fff1d183d633f7b8809b0aae96d1737ce1a4..642afddd88c154f86bce47622b9d59cc5aec446e 100644
--- a/clang/test/Sema/aarch64-neon-target.c
+++ b/clang/test/Sema/aarch64-neon-target.c
@@ -69,8 +69,8 @@ void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t
   vrnd_f16(v4f16); // expected-error {{always_inline function 'vrnd_f16' requires target feature 'fullfp16'}}
   vmaxnm_f16(v4f16, v4f16); // expected-error {{always_inline function 'vmaxnm_f16' requires target feature 'fullfp16'}}
   vrndi_f16(v4f16); // expected-error {{always_inline function 'vrndi_f16' requires target feature 'fullfp16'}}
-  // fp16fml
-  vfmlal_low_f16(v2f32, v4f16, v4f16); // expected-error {{always_inline function 'vfmlal_low_f16' requires target feature 'fp16fml'}}
+  // fp16fml depends on fp-armv8
+  vfmlal_low_f16(v2f32, v4f16, v4f16); // expected-error {{always_inline function 'vfmlal_low_f16' requires target feature 'fp-armv8'}}
   // i8mm
   vmmlaq_s32(v4i32, v8i16, v8i16); // expected-error {{always_inline function 'vmmlaq_s32' requires target feature 'i8mm'}}
   vusdot_laneq_s32(v2i32, v8i8, v8i16, 0); // expected-error {{always_inline function 'vusdot_s32' requires target feature 'i8mm'}}
diff --git a/clang/test/Sema/attr-availability-macosx.cpp b/clang/test/Sema/attr-availability-macosx.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..52f320d4092815b39abb5dd09ab5d5f35c601c73
--- /dev/null
+++ b/clang/test/Sema/attr-availability-macosx.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 "-triple" "arm64-apple-macosx10.15" -fsyntax-only -verify %s
+
+__attribute__((availability(macos,introduced=11)))
+inline bool try_acquire() {
+  return true;
+}
+
+template <class T>
+__attribute__((availability(macos,introduced=11)))
+bool try_acquire_for(T duration) { // expected-note{{'try_acquire_for<int>' has been marked as being introduced in macOS 11 here, but the deployment target is macOS 10.15}}
+  return try_acquire();
+}
+
+int main() {
+  try_acquire_for(1); // expected-warning{{'try_acquire_for<int>' is only available on macOS 11 or newer}}
+  // expected-note@-1{{enclose 'try_acquire_for<int>' in a __builtin_available check to silence this warning}}
+}
\ No newline at end of file
diff --git a/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp b/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2b8fac72847d6a2d691e82d8d85785912c63a0eb
--- /dev/null
+++ b/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 %s -x hip -fcuda-is-device -o - \
+// RUN:   -triple=spirv64-amd-amdhsa -fsyntax-only \
+// RUN:   -verify=dev
+// RUN: %clang_cc1 %s -x hip -triple x86_64 -o - \
+// RUN:   -aux-triple spirv64-amd-amdhsa -fsyntax-only \
+// RUN:   -verify=host
+
+// dev-no-diagnostics
+
+void test_host() {
+  __UINT32_TYPE__ val32;
+  __UINT64_TYPE__ val64;
+
+  // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_inc32' in __host__ function}}
+  val32 = __builtin_amdgcn_atomic_inc32(&val32, val32, __ATOMIC_SEQ_CST, "");
+
+  // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_inc64' in __host__ function}}
+  val64 = __builtin_amdgcn_atomic_inc64(&val64, val64, __ATOMIC_SEQ_CST, "");
+
+  // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_dec32' in __host__ function}}
+  val32 = __builtin_amdgcn_atomic_dec32(&val32, val32, __ATOMIC_SEQ_CST, "");
+
+  // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_dec64' in __host__ function}}
+  val64 = __builtin_amdgcn_atomic_dec64(&val64, val64, __ATOMIC_SEQ_CST, "");
+}
diff --git a/clang/test/Sema/constexpr-void-cast.c b/clang/test/Sema/constexpr-void-cast.c
index 91e4027f67fe387850891099e773b490d74e5c26..2ffc59f509b4b72b67a4660ef2e393d1254a573a 100644
--- a/clang/test/Sema/constexpr-void-cast.c
+++ b/clang/test/Sema/constexpr-void-cast.c
@@ -1,8 +1,12 @@
 // RUN: %clang_cc1 -x c -fsyntax-only %s -verify=c -std=c11
 // RUN: %clang_cc1 -x c -fsyntax-only %s -pedantic -verify=c-pedantic -std=c11
+// RUN: %clang_cc1 -x c -fsyntax-only %s -verify=c -std=c11 -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -x c -fsyntax-only %s -pedantic -verify=c-pedantic -std=c11 -fexperimental-new-constant-interpreter
 //
 // RUN: %clang_cc1 -x c++ -fsyntax-only %s -verify=cxx
 // RUN: %clang_cc1 -x c++ -fsyntax-only %s -pedantic -verify=cxx-pedantic
+// RUN: %clang_cc1 -x c++ -fsyntax-only %s -verify=cxx -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -x c++ -fsyntax-only %s -pedantic -verify=cxx-pedantic -fexperimental-new-constant-interpreter
 
 // c-no-diagnostics
 // cxx-no-diagnostics
diff --git a/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl b/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl
new file mode 100644
index 0000000000000000000000000000000000000000..0fb1b5f3672265117fc52cd400898de3c6f9b52d
--- /dev/null
+++ b/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl
@@ -0,0 +1,111 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -fsyntax-only -verify %s
+
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+kernel void test () {
+
+  int sgpr = 0, vgpr = 0, imm = 0;
+
+  // sgpr constraints
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "s" (imm) : );
+
+  __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec}" (imm) : );
+  __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exe" (imm) : ); // expected-error {{invalid input constraint '{exe' in asm}}
+  __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec" (imm) : ); // expected-error {{invalid input constraint '{exec' in asm}}
+  __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec}a" (imm) : ); // expected-error {{invalid input constraint '{exec}a' in asm}}
+
+  // vgpr constraints
+  __asm__ ("v_mov_b32 %0, %1" : "=v" (vgpr) : "v" (imm) : );
+
+  // 'I' constraint (an immediate integer in the range -16 to 64)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (imm) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-16) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (64) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-17) : ); // expected-error {{value '-17' out of range for constraint 'I'}}
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (65) : ); // expected-error {{value '65' out of range for constraint 'I'}}
+
+  // 'J' constraint (an immediate 16-bit signed integer)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (imm) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32768) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32767) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32769) : ); // expected-error {{value '-32769' out of range for constraint 'J'}}
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32768) : ); // expected-error {{value '32768' out of range for constraint 'J'}}
+
+  // 'A' constraint (an immediate constant that can be inlined)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "A" (imm) : );
+
+  // 'B' constraint (an immediate 32-bit signed integer)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "B" (imm) : );
+
+  // 'C' constraint (an immediate 32-bit unsigned integer or 'A' constraint)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "C" (imm) : );
+
+  // 'DA' constraint (an immediate 64-bit constant that can be split into two 'A' constants)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DA" (imm) : );
+
+  // 'DB' constraint (an immediate 64-bit constant that can be split into two 'B' constants)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DB" (imm) : );
+
+}
+
+__kernel void
+test_float(const __global float *a, const __global float *b, __global float *c, unsigned i)
+{
+    float ai = a[i];
+    float bi = b[i];
+    float ci;
+
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{v2}"(ai), "{v3}"(bi) : );
+    __asm("v_add_f32_e32 v1, v2, v3" : ""(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "="(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '=' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={a}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={a}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1a}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1a}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={va}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={va}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1}a"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1}a' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "=v1}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '=v1}' in asm}}
+
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v[1]}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : );
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v[1}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[1}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v[1]"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[1]' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v[a]}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[a]}' in asm}}
+
+    __asm("v_add_f32_e32 v1, v2, v3" : "=v"(ci) : "v"(ai), "v"(bi) : );
+    __asm("v_add_f32_e32 v1, v2, v3" : "=v1"(ci) : "v2"(ai), "v3"(bi) : ); /// expected-error {{invalid output constraint '=v1' in asm}}
+
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{a}"(ai), "{v3}"(bi) : ); // expected-error {{invalid input constraint '{a}' in asm}}
+    __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{v2}"(ai), "{a}"(bi) : ); // expected-error {{invalid input constraint '{a}' in asm}}
+    c[i] = ci;
+}
+
+__kernel void
+test_double(const __global double *a, const __global double *b, __global double *c, unsigned i)
+{
+    double ai = a[i];
+    double bi = b[i];
+    double ci;
+
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : );
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "=v{[1:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '=v{[1:2]}' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]a}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]a}' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]}a"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]}a' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:]}' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[:2]}' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2}' in asm}}
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[2:1]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[2:1]}' in asm}}
+
+    __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "=v[1:2]"(ci) : "v[3:4]"(ai), "v[5:6]"(bi) : ); //expected-error {{invalid output constraint '=v[1:2]' in asm}}
+
+    c[i] = ci;
+}
+
+void test_long(int arg0) {
+  long v15_16;
+  __asm volatile("v_lshlrev_b64 v[15:16], 0, %0" : "={v[15:16]}"(v15_16) : "v"(arg0));
+}
diff --git a/clang/test/SemaCUDA/allow-int128.cu b/clang/test/SemaCUDA/allow-int128.cu
index eb7b7e7f52862b38acc64d7c364e350041b1e3c3..af3e8c2453ad184d71648ff25cff211668ed3c4f 100644
--- a/clang/test/SemaCUDA/allow-int128.cu
+++ b/clang/test/SemaCUDA/allow-int128.cu
@@ -1,6 +1,9 @@
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa \
 // RUN:   -aux-triple x86_64-unknown-linux-gnu \
 // RUN:   -fcuda-is-device -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa \
+// RUN:   -aux-triple x86_64-unknown-linux-gnu \
+// RUN:   -fcuda-is-device -verify -fsyntax-only %s
 // RUN: %clang_cc1 -triple nvptx \
 // RUN:   -aux-triple x86_64-unknown-linux-gnu \
 // RUN:   -fcuda-is-device -verify -fsyntax-only %s
diff --git a/clang/test/SemaCUDA/amdgpu-f128.cu b/clang/test/SemaCUDA/amdgpu-f128.cu
index 9a0212cdb93cffa78b0f9bc0d27c547764cabff9..1f5a6553dcc4fdc0f3d6ea10ce5288e652114b1d 100644
--- a/clang/test/SemaCUDA/amdgpu-f128.cu
+++ b/clang/test/SemaCUDA/amdgpu-f128.cu
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -fsyntax-only -verify %s
 
 // expected-no-diagnostics
 typedef __float128 f128_t;
diff --git a/clang/test/SemaCUDA/float16.cu b/clang/test/SemaCUDA/float16.cu
index bb5ed6064384918803db62310f46abb46f6bfa8a..9c7faef284fee76d96107d64d4523ab05d2d6394 100644
--- a/clang/test/SemaCUDA/float16.cu
+++ b/clang/test/SemaCUDA/float16.cu
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple amdgcn -verify %s
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple spirv64-amd-amdhsa -verify %s
 // RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple nvptx64 -verify %s
 // expected-no-diagnostics
 #include "Inputs/cuda.h"
diff --git a/clang/test/SemaCUDA/fp16-arg-return.cu b/clang/test/SemaCUDA/fp16-arg-return.cu
index 46d543f44445dc8b2cf9c424ca80a6a508e26286..9347491caa97b9e69bac292abf4446b46b4c06a7 100644
--- a/clang/test/SemaCUDA/fp16-arg-return.cu
+++ b/clang/test/SemaCUDA/fp16-arg-return.cu
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -o - -triple amdgcn-amd-amdhsa -fcuda-is-device -fsyntax-only -verify %s
+// RUN: %clang_cc1 -o - -triple spirv64-amd-amdhsa -fcuda-is-device -fsyntax-only -verify %s
 
 // expected-no-diagnostics
 
diff --git a/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu b/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu
new file mode 100644
index 0000000000000000000000000000000000000000..ea1f24670ff9a815c0b84824776dab48ffab1ea6
--- /dev/null
+++ b/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu
@@ -0,0 +1,86 @@
+// RUN: %clang_cc1 -x hip -std=c++11 -triple spirv64-amd-amdhsa -fcuda-is-device -verify -fsyntax-only %s
+
+#include "Inputs/cuda.h"
+
+__device__ int test_hip_atomic_load(int *pi32, unsigned int *pu32, long long *pll, unsigned long long *pull, float *fp, double *dbl) {
+  int val = __hip_atomic_load(0);      // expected-error {{too few arguments to function call, expected 3, have 1}}
+  val = __hip_atomic_load(0, 0, 0, 0); // expected-error {{too many arguments to function call, expected 3, have 4}}
+  val = __hip_atomic_load(0, 0, 0);    // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+  val = __hip_atomic_load(pi32, 0, 0); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT);
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP);
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, 6); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pi32, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pi32, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pi32, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pi32, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}}
+  val = __hip_atomic_load(pu32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pll, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(pull, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(fp, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  val = __hip_atomic_load(dbl, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  return val;
+}
+
+__device__ int test_hip_atomic_store(int *pi32, unsigned int *pu32, long long *pll, unsigned long long *pull, float *fp, double *dbl,
+                                     int i32, unsigned int u32, long long i64, unsigned long long u64, float f32, double f64) {
+  __hip_atomic_store(0);             // expected-error {{too few arguments to function call, expected 4, have 1}}
+  __hip_atomic_store(0, 0, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+  __hip_atomic_store(0, 0, 0, 0);    // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+  __hip_atomic_store(pi32, 0, 0, 0); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT);
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP);
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, 6); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, 0, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, 0, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}}
+  __hip_atomic_store(pi32, 0, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}}
+  __hip_atomic_store(pi32, 0, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}}
+  __hip_atomic_store(pi32, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pu32, u32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pll, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pull, u64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(fp, f32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(dbl, f64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, u32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pi32, u64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(pll, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(fp, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(fp, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(dbl, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  __hip_atomic_store(dbl, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  return 0;
+}
+
+__device__ bool test_hip_atomic_cmpxchg_weak(int *ptr, int val, int desired) {
+  bool flag = __hip_atomic_compare_exchange_weak(0);                                     // expected-error {{too few arguments to function call, expected 6, have 1}}
+  flag = __hip_atomic_compare_exchange_weak(0, 0, 0, 0, 0, 0, 0);                        // expected-error {{too many arguments to function call, expected 6, have 7}}
+  flag = __hip_atomic_compare_exchange_weak(0, 0, 0, 0, 0, 0);                           // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+  flag = __hip_atomic_compare_exchange_weak(ptr, 0, 0, 0, 0, 0);                         // expected-error {{synchronization scope argument to atomic operation is invalid}}, expected-warning {{null passed to a callee that requires a non-null argument}}
+  flag = __hip_atomic_compare_exchange_weak(ptr, 0, 0, 0, 0, __HIP_MEMORY_SCOPE_SYSTEM); // expected-warning {{null passed to a callee that requires a non-null argument}}
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_CONSUME, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning {{failure memory order argument to atomic operation is invalid}}
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_CONSUME, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD);
+  return flag;
+}
diff --git a/clang/test/SemaCXX/PR8755.cpp b/clang/test/SemaCXX/PR8755.cpp
index 6818f3f0a82276fe4f52794cc82cbe9836ebf5a2..c0bcab3537d654691fbb1070ba147c9285518534 100644
--- a/clang/test/SemaCXX/PR8755.cpp
+++ b/clang/test/SemaCXX/PR8755.cpp
@@ -7,7 +7,7 @@ struct A {
 
 template <typename T>
 void f() {
-  class A <T> ::iterator foo;  // expected-error{{typedef 'iterator' cannot be referenced with a class specifier}}
+  class A <T> ::iterator foo;  // expected-error{{typedef 'iterator' cannot be referenced with the 'class' specifier}}
 }
 
 void g() {
diff --git a/clang/test/SemaCXX/attr-weak.cpp b/clang/test/SemaCXX/attr-weak.cpp
index f065bfd9483f8a127f4add93d6ecf988f4b5d08d..0f9a2975e5f68e8f3167ef3d6dad853febe021a1 100644
--- a/clang/test/SemaCXX/attr-weak.cpp
+++ b/clang/test/SemaCXX/attr-weak.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -verify -std=c++11 %s -fexperimental-new-constant-interpreter
 
 static int test0 __attribute__((weak)); // expected-error {{weak declaration cannot have internal linkage}}
 static void test1() __attribute__((weak)); // expected-error {{weak declaration cannot have internal linkage}}
diff --git a/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp b/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d47a39a0754c54f1f7659304e0affd1747323391
--- /dev/null
+++ b/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -DSANITIZER_ENABLED -fsanitize=address -fsanitize-address-field-padding=1 %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux %s
+
+struct S {
+  ~S() {}
+  virtual void foo() {}
+
+  int buffer[1];
+  int other_field = 0;
+};
+
+union U {
+  S s;
+};
+
+struct Derived : S {};
+
+static_assert(!__is_trivially_copyable(S));
+#ifdef SANITIZER_ENABLED
+// Don't allow memcpy when the struct has poisoned padding bits.
+// The sanitizer adds posion padding bits to struct S.
+static_assert(sizeof(S) > 16);
+static_assert(!__is_bitwise_cloneable(S));
+static_assert(sizeof(U) == sizeof(S)); // no padding bit for U.
+static_assert(!__is_bitwise_cloneable(U));
+static_assert(!__is_bitwise_cloneable(S[2]));
+static_assert(!__is_bitwise_cloneable(Derived));
+#else
+static_assert(sizeof(S) == 16);
+static_assert(__is_bitwise_cloneable(S));
+static_assert(__is_bitwise_cloneable(U));
+static_assert(__is_bitwise_cloneable(S[2]));
+static_assert(__is_bitwise_cloneable(Derived));
+#endif
diff --git a/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp b/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1781cf48449f6218b7eff0876093a56573c0d0c0
--- /dev/null
+++ b/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+//
+struct DynamicClass { virtual int Foo(); };
+static_assert(!__is_trivially_copyable(DynamicClass));
+static_assert(__is_bitwise_cloneable(DynamicClass));
+
+struct InComplete; // expected-note{{forward declaration}}
+static_assert(!__is_bitwise_cloneable(InComplete)); // expected-error{{incomplete type 'InComplete' used in type trait expression}}
diff --git a/clang/test/SemaCXX/complex-folding.cpp b/clang/test/SemaCXX/complex-folding.cpp
index 054f159e9ce0dd2dd79b00f1a355f78a0b318847..7bfd36f156ea61e1f99d6b01f6bb00fc94876ca3 100644
--- a/clang/test/SemaCXX/complex-folding.cpp
+++ b/clang/test/SemaCXX/complex-folding.cpp
@@ -59,41 +59,48 @@ static_assert((1.25 / (0.25 - 0.75j)) == (0.5 + 1.5j));
 
 // Test that infinities are preserved, don't turn into NaNs, and do form zeros
 // when the divisor.
+constexpr _Complex float InfC = {1.0, __builtin_inf()};
+constexpr _Complex float InfInf = __builtin_inf() + InfC;
+static_assert(__real__(InfInf) == __builtin_inf());
+static_assert(__imag__(InfInf) == __builtin_inf());
+static_assert(__builtin_isnan(__real__(InfInf * InfInf)));
+static_assert(__builtin_isinf_sign(__imag__(InfInf * InfInf)) == 1);
+
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) * 1.0)) == 1);
-static_assert(__builtin_isinf_sign(__imag__((1.0 + __builtin_inf() * 1.0j) * 1.0)) == 1);
+static_assert(__builtin_isinf_sign(__imag__((1.0 + InfC) * 1.0)) == 1);
 static_assert(__builtin_isinf_sign(__real__(1.0 * (__builtin_inf() + 1.0j))) == 1);
-static_assert(__builtin_isinf_sign(__imag__(1.0 * (1.0 + __builtin_inf() * 1.0j))) == 1);
-
+static_assert(__builtin_isinf_sign(__imag__(1.0 * (1.0 + InfC))) == 1);
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) * (1.0 + 1.0j))) == 1);
 static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) * (__builtin_inf() + 1.0j))) == 1);
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) * (__builtin_inf() + 1.0j))) == 1);
-
-static_assert(__builtin_isinf_sign(__real__((1.0 + __builtin_inf() * 1.0j) * (1.0 + 1.0j))) == -1);
-static_assert(__builtin_isinf_sign(__imag__((1.0 + __builtin_inf() * 1.0j) * (1.0 + 1.0j))) == 1);
-static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) * (1.0 + __builtin_inf() * 1.0j))) == -1);
-static_assert(__builtin_isinf_sign(__imag__((1.0 + 1.0j) * (1.0 + __builtin_inf() * 1.0j))) == 1);
-
-static_assert(__builtin_isinf_sign(__real__((1.0 + __builtin_inf() * 1.0j) * (1.0 + __builtin_inf() * 1.0j))) == -1);
-static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + __builtin_inf() * 1.0j) * (__builtin_inf() + __builtin_inf() * 1.0j))) == -1);
-
+static_assert(__builtin_isinf_sign(__real__((1.0 + InfC) * (1.0 + 1.0j))) == -1);
+static_assert(__builtin_isinf_sign(__imag__((1.0 + InfC) * (1.0 + 1.0j))) == 1);
+static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) * (1.0 + InfC))) == -1);
+static_assert(__builtin_isinf_sign(__imag__((1.0 + 1.0j) * (1.0 + InfC))) == 1);
+static_assert(__builtin_isinf_sign(__real__((1.0 + InfC) * (1.0 + InfC))) == -1);
+static_assert(__builtin_isinf_sign(__real__(InfInf * InfInf)) == 0);
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) / (1.0 + 1.0j))) == 1);
-static_assert(__builtin_isinf_sign(__imag__(1.0 + (__builtin_inf() * 1.0j) / (1.0 + 1.0j))) == 1);
-static_assert(__builtin_isinf_sign(__imag__((__builtin_inf() + __builtin_inf() * 1.0j) / (1.0 + 1.0j))) == 1);
+static_assert(__builtin_isinf_sign(__imag__(1.0 + (InfC) / (1.0 + 1.0j))) == 1);
+static_assert(__builtin_isinf_sign(__imag__((InfInf) / (1.0 + 1.0j))) == 0);
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) / 1.0)) == 1);
-static_assert(__builtin_isinf_sign(__imag__(1.0 + (__builtin_inf() * 1.0j) / 1.0)) == 1);
-static_assert(__builtin_isinf_sign(__imag__((__builtin_inf() + __builtin_inf() * 1.0j) / 1.0)) == 1);
-
+static_assert(__builtin_isinf_sign(__imag__(1.0 + (InfC) / 1.0)) == 1);
+static_assert(__builtin_isinf_sign(__imag__((InfInf) / 1.0)) == 1);
 static_assert(((1.0 + 1.0j) / (__builtin_inf() + 1.0j)) == (0.0 + 0.0j));
-static_assert(((1.0 + 1.0j) / (1.0 + __builtin_inf() * 1.0j)) == (0.0 + 0.0j));
-static_assert(((1.0 + 1.0j) / (__builtin_inf() + __builtin_inf() * 1.0j)) == (0.0 + 0.0j));
+static_assert(((1.0 + 1.0j) / (1.0 + InfC)) == (0.0 + 0.0j));
+static_assert(((1.0 + 1.0j) / (InfInf)) == (0.0 + 0.0j));
 static_assert(((1.0 + 1.0j) / __builtin_inf()) == (0.0 + 0.0j));
-
+static_assert(1.0j / 0.0 == 1); // expected-error {{static assertion}} \
+                                // expected-note {{division by zero}}
 static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) / (0.0 + 0.0j))) == 1);
-static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) / 0.0)) == 1);
-
+static_assert(__builtin_isinf_sign(__real__((1.0 + 1.0j) / 0.0)) == 1); // expected-error {{static assertion}} \
+                                                                        // expected-note {{division by zero}}
 static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) / (0.0 + 0.0j))) == 1);
-static_assert(__builtin_isinf_sign(__imag__((1.0 + __builtin_inf() * 1.0j) / (0.0 + 0.0j))) == 1);
-static_assert(__builtin_isinf_sign(__imag__((__builtin_inf() + __builtin_inf() * 1.0j) / (0.0 + 0.0j))) == 1);
-static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) / 0.0)) == 1);
-static_assert(__builtin_isinf_sign(__imag__((1.0 + __builtin_inf() * 1.0j) / 0.0)) == 1);
-static_assert(__builtin_isinf_sign(__imag__((__builtin_inf() + __builtin_inf() * 1.0j) / 0.0)) == 1);
+static_assert(__builtin_isinf_sign(__imag__((1.0 + InfC) / (0.0 + 0.0j))) == 1);
+static_assert(__builtin_isinf_sign(__imag__((InfInf) / (0.0 + 0.0j))) == 1);
+static_assert(__builtin_isinf_sign(__real__((__builtin_inf() + 1.0j) / 0.0)) == 1); // expected-error {{static assertion}} \
+                                                                                    // expected-note {{division by zero}}
+static_assert(__builtin_isinf_sign(__imag__((1.0 + InfC) / 0.0)) == 1); // expected-error {{static assertion}} \
+                                                                        // expected-note {{division by zero}}
+static_assert(__builtin_isinf_sign(__imag__((InfInf) / 0.0)) == 1); // expected-error {{static assertion}} \
+                                                                    // expected-note {{division by zero}}
+
diff --git a/clang/test/SemaCXX/constant-expression-cxx14.cpp b/clang/test/SemaCXX/constant-expression-cxx14.cpp
index 80a7a2dd31531c4fbd933745be3390b36a7af8dc..70ab5dcd357c1caa4d6d6adaf38b74c55b36dea7 100644
--- a/clang/test/SemaCXX/constant-expression-cxx14.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx14.cpp
@@ -82,7 +82,7 @@ constexpr void k() {
 
 // If the return type is not 'void', no return statements => never a constant
 // expression, so still diagnose that case.
-[[noreturn]] constexpr int fn() { // expected-error {{no return statement in constexpr function}}
+[[noreturn]] constexpr int fn() { // cxx14_20-error {{no return statement in constexpr function}}
   fn();
 }
 
diff --git a/clang/test/SemaCXX/constexpr-default-arg.cpp b/clang/test/SemaCXX/constexpr-default-arg.cpp
index 901123bfb359ff17bba26b8d4eb7fffe384ed333..ec9b2927880bdfb69c4c71dde805b00fbf26bcf5 100644
--- a/clang/test/SemaCXX/constexpr-default-arg.cpp
+++ b/clang/test/SemaCXX/constexpr-default-arg.cpp
@@ -32,8 +32,8 @@ void test_default_arg2() {
 }
 
 // Check that multiple CXXDefaultInitExprs don't cause an assertion failure.
-struct A { int &&r = 0; };
+struct A { int &&r = 0; }; // expected-note 2{{default member initializer}}
 struct B { A x, y; };
-B b = {}; // expected-no-diagnostics
+B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
 
 }
diff --git a/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp b/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..25d1f8df7f71663a99dbacc022e1b9f9db821ff7
--- /dev/null
+++ b/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s
+
+constexpr int f() { } // expected-warning {{non-void function does not return a value}}
+static_assert(__is_same(decltype([] constexpr -> int { }( )), int)); // expected-warning {{non-void lambda does not return a value}}
+
+consteval int g() { } // expected-warning {{non-void function does not return a value}}
+static_assert(__is_same(decltype([] consteval -> int { }( )), int)); // expected-warning {{non-void lambda does not return a value}}
diff --git a/clang/test/SemaCXX/cxx11-default-member-initializers.cpp b/clang/test/SemaCXX/cxx11-default-member-initializers.cpp
index 1ea8b98cd86367f41c1927b24e7b523fd07e6835..dd8e9c6b7fc11f0384840513a17afa4723cb484e 100644
--- a/clang/test/SemaCXX/cxx11-default-member-initializers.cpp
+++ b/clang/test/SemaCXX/cxx11-default-member-initializers.cpp
@@ -27,80 +27,6 @@ class MemInit {
   C m = s;
 };
 
-namespace std {
-typedef decltype(sizeof(int)) size_t;
-
-// libc++'s implementation
-template <class _E> class initializer_list {
-  const _E *__begin_;
-  size_t __size_;
-
-  initializer_list(const _E *__b, size_t __s) : __begin_(__b), __size_(__s) {}
-
-public:
-  typedef _E value_type;
-  typedef const _E &reference;
-  typedef const _E &const_reference;
-  typedef size_t size_type;
-
-  typedef const _E *iterator;
-  typedef const _E *const_iterator;
-
-  initializer_list() : __begin_(nullptr), __size_(0) {}
-
-  size_t size() const { return __size_; }
-  const _E *begin() const { return __begin_; }
-  const _E *end() const { return __begin_ + __size_; }
-};
-} // namespace std
-
-#if __cplusplus >= 201703L
-namespace test_rebuild {
-template <typename T, int> class C {
-public:
-  C(std::initializer_list<T>);
-};
-
-template <typename T> using Ptr = __remove_pointer(T) *;
-template <typename T> C(T) -> C<Ptr<T>, sizeof(T)>;
-
-class A {
-public:
-  template <typename T1, typename T2> T1 *some_func(T2 &&);
-};
-
-struct B : A {
-  // Test CXXDefaultInitExpr rebuild issue in 
-  // https://github.com/llvm/llvm-project/pull/87933
-  int *ar = some_func<int>(C{some_func<int>(0)});
-  B() {}
-};
-
-int TestBody_got;
-template <int> class Vector {
-public:
-  Vector(std::initializer_list<int>);
-};
-template <typename... Ts> Vector(Ts...) -> Vector<sizeof...(Ts)>;
-class ProgramBuilder {
-public:
-  template <typename T, typename ARGS> int *create(ARGS);
-};
-
-struct TypeTest : ProgramBuilder {
-  int *str_f16 = create<int>(Vector{0});
-  TypeTest() {}
-};
-class TypeTest_Element_Test : TypeTest {
-  void TestBody();
-};
-void TypeTest_Element_Test::TestBody() {
-  int *expect = str_f16;
-  &TestBody_got != expect; // expected-warning {{inequality comparison result unused}}
-}
-} //  namespace test_rebuild
-#endif // __cplusplus >= 201703L
-
 #if __cplusplus >= 202002L
 // This test ensures cleanup expressions are correctly produced
 // in the presence of default member initializers.
diff --git a/clang/test/SemaCXX/eval-crashes.cpp b/clang/test/SemaCXX/eval-crashes.cpp
index a06f60f71e9c7ee056a3ccbfdc203778741784b9..017df977b26b7b08637d39c5943ee1634b8d6592 100644
--- a/clang/test/SemaCXX/eval-crashes.cpp
+++ b/clang/test/SemaCXX/eval-crashes.cpp
@@ -25,9 +25,11 @@ namespace pr33140_0b {
 }
 
 namespace pr33140_2 {
-  struct A { int &&r = 0; };
+  // FIXME: The declaration of 'b' below should lifetime-extend two int
+  // temporaries.
+  struct A { int &&r = 0; }; // expected-note 2{{initializing field 'r' with default member initializer}}
   struct B { A x, y; };
-  B b = {};
+  B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
 }
 
 namespace pr33140_3 {
diff --git a/clang/test/SemaCXX/for-range-examples.cpp b/clang/test/SemaCXX/for-range-examples.cpp
index d129d50d673e162da8f4ea6a7316de7d413c1c9f..c06bf0102bcfead12199b0071ed35fa0cc70d515 100644
--- a/clang/test/SemaCXX/for-range-examples.cpp
+++ b/clang/test/SemaCXX/for-range-examples.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++11
+// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++11 -fexperimental-new-constant-interpreter
 
 namespace value_range_detail {
   template<typename T>
diff --git a/clang/test/SemaCXX/integer-overflow.cpp b/clang/test/SemaCXX/integer-overflow.cpp
index 6049458b93bbd094a10eb60e5d13ca398d706c04..d1cc8bee566f6b0409e75bebcc5c8bdb1e7234d0 100644
--- a/clang/test/SemaCXX/integer-overflow.cpp
+++ b/clang/test/SemaCXX/integer-overflow.cpp
@@ -1,6 +1,10 @@
 // RUN: %clang_cc1 %s -verify -fsyntax-only -std=gnu++98 -triple x86_64-pc-linux-gnu
 // RUN: %clang_cc1 %s -verify -fsyntax-only -std=gnu++2a -triple x86_64-pc-linux-gnu
 
+// RUN: %clang_cc1 %s -verify -fsyntax-only -std=gnu++98 -triple x86_64-pc-linux-gnu -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 %s -verify -fsyntax-only -std=gnu++2a -triple x86_64-pc-linux-gnu -fexperimental-new-constant-interpreter
+
+
 typedef unsigned long long uint64_t;
 typedef unsigned int uint32_t;
 
diff --git a/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp b/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp
index 6273d9c42e0b153fe319e6d41220fec9f75e1ff6..98bec184164b63b012e933d01439640ec69f8712 100644
--- a/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp
+++ b/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -Wno-tautological-pointer-compare -fblocks -std=c++11 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -Wno-tautological-pointer-compare -fblocks -std=c++11 -verify %s -fexperimental-new-constant-interpreter
 
 void foo() {
   int a;
diff --git a/clang/test/SemaCXX/using-decl-templates.cpp b/clang/test/SemaCXX/using-decl-templates.cpp
index 77dc596fdfc9f33fd65b6ff6aed65352e2de8ac8..1cf4caee1c0db0dbeca52090a6917a408bfff486 100644
--- a/clang/test/SemaCXX/using-decl-templates.cpp
+++ b/clang/test/SemaCXX/using-decl-templates.cpp
@@ -90,7 +90,7 @@ namespace aliastemplateinst {
   template<typename T> struct A { };
   template<typename T> using APtr = A<T*>; // expected-note{{previous use is here}}
 
-  template struct APtr<int>; // expected-error{{type alias template 'APtr' cannot be referenced with a struct specifier}}
+  template struct APtr<int>; // expected-error{{alias template 'APtr' cannot be referenced with the 'struct' specifier}}
 }
 
 namespace DontDiagnoseInvalidTest {
diff --git a/clang/test/SemaObjCXX/arc-type-traits.mm b/clang/test/SemaObjCXX/arc-type-traits.mm
index 2d30ae450f3b096090c280e01e4d9786f7206967..25bc8b362140a603c173c1a65ac080b5a2f6c7a3 100644
--- a/clang/test/SemaObjCXX/arc-type-traits.mm
+++ b/clang/test/SemaObjCXX/arc-type-traits.mm
@@ -221,3 +221,12 @@ TRAIT_IS_TRUE(__is_trivially_relocatable, __unsafe_unretained id);
 TRAIT_IS_TRUE(__is_trivially_relocatable, HasStrong);
 TRAIT_IS_FALSE(__is_trivially_relocatable, HasWeak);
 TRAIT_IS_TRUE(__is_trivially_relocatable, HasUnsafeUnretained);
+
+// __is_bitwise_cloneable
+TRAIT_IS_FALSE(__is_bitwise_cloneable, __strong id);
+TRAIT_IS_FALSE(__is_bitwise_cloneable, __weak id);
+TRAIT_IS_FALSE(__is_bitwise_cloneable, __autoreleasing id);
+TRAIT_IS_TRUE(__is_trivial, __unsafe_unretained id);
+TRAIT_IS_FALSE(__is_bitwise_cloneable, HasStrong);
+TRAIT_IS_FALSE(__is_bitwise_cloneable, HasWeak);
+TRAIT_IS_TRUE(__is_bitwise_cloneable, HasUnsafeUnretained);
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl
index bb949c0ddd10df152c07682b7ffa2905c2230633..969ff4ba9c9203fff97ae033bafc69b0416ab16a 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx908 -verify -S -o - %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s
 
 #pragma OPENCL EXTENSION cl_khr_fp64:enable
 
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl
index 701016148a893cd110d3d9fb3fc86560b1f0cac1..235fa8263140258a8d8586499c80d761e004a8ba 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx90a -verify -S -o - %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s
 
 #pragma OPENCL EXTENSION cl_khr_fp64:enable
 
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl
index b177b93938e46e52fb813b5809673459b99fbd7d..0fc2304d51ce0571285478bf342faafb0a37e1e8 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx940 -verify -S -o - %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s
 
 typedef float  v2f   __attribute__((ext_vector_type(2)));
 typedef float  v4f   __attribute__((ext_vector_type(4)));
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl b/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl
index 487cc53e8ad8a375d2486d54a6bbdb9761aebc71..2a1ba4300864cb4325181bcc8f813a0985c4b792 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl
@@ -3,8 +3,10 @@
 
 typedef unsigned int u32;
 
-void test_global_load_lds_unsupported_size(global u32* src, local u32 *dst, u32 size) {
-  __builtin_amdgcn_global_load_lds(src, dst, size, /*offset=*/0, /*aux=*/0); // expected-error{{expression is not an integer constant expression}}
+void test_global_load_lds_unsupported_size(global u32* src, local u32 *dst, u32 size, u32 offset, u32 aux) {
+  __builtin_amdgcn_global_load_lds(src, dst, size, /*offset=*/0, /*aux=*/0); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}}
+  __builtin_amdgcn_global_load_lds(src, dst, /*size=*/4, offset, /*aux=*/0); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}}
+  __builtin_amdgcn_global_load_lds(src, dst, /*size=*/4, /*offset=*/0, aux); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}}
   __builtin_amdgcn_global_load_lds(src, dst, /*size=*/5, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}}
   __builtin_amdgcn_global_load_lds(src, dst, /*size=*/0, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}}
   __builtin_amdgcn_global_load_lds(src, dst, /*size=*/3, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}}
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index 45e74cce3a98c851bc3bd9d279a719d0c159ad95..7675d4287cb88a49654f1966115d95440cfbe20f 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -200,4 +200,132 @@ namespace consistency {
     template struct A<B<int>, B<int>, B<int>>;
     // new-error@-1 {{ambiguous partial specializations}}
   } // namespace t2
+  namespace t3 {
+    template<class T1, class T2, class T3> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3, class T4>
+    struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T2>>::type> {};
+    // new-note@-1 {{partial specialization matches}}
+
+    template<template<class> class UU1,
+             class U1, class U2>
+    struct A<UU1<U1>, UU1<U2>, typename nondeduced<UU1<U1>>::type>;
+    // new-note@-1 {{partial specialization matches}}
+
+    template struct A<B<int>, B<int>, B<int>>;
+    // new-error@-1 {{ambiguous partial specializations}}
+  } // namespace t3
+  namespace t4 {
+    template<class T1, class T2, class T3> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3, class T4>
+    struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T4>>::type> {};
+    // new-note@-1 {{partial specialization matches}}
+
+    template<template<class> class UU1,
+             class U1, class U2>
+    struct A<UU1<U1>, UU1<U2>, typename nondeduced<UU1<U1>>::type>;
+    // new-note@-1 {{partial specialization matches}}
+
+    template struct A<B<int>, B<int>, B<int>>;
+    // new-error@-1 {{ambiguous partial specializations}}
+  } // namespace t4
+  namespace t5 {
+    template<class T1, class T2> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3, class T4>
+    struct A<TT1<T1, T2>, TT1<T3, T4>> {};
+    // new-note@-1 {{partial specialization matches}}
+
+    template<template<class> class UU1,
+             class U1, class U2>
+    struct A<UU1<U1>, UU1<U2>>;
+    // new-note@-1 {{partial specialization matches}}
+
+    template struct A<B<int>, B<int>>;
+    // new-error@-1 {{ambiguous partial specializations}}
+  } // namespace t5
+  namespace t6 {
+    template<class T1, class T2> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3>
+    struct A<TT1<T1, T2>, TT1<T1, T3>> {};
+    // new-note@-1 {{partial specialization matches}}
+
+    template<template<class> class UU1,
+             class U1, class U2>
+    struct A<UU1<U1>, UU1<U2>>;
+    // new-note@-1 {{partial specialization matches}}
+
+    template struct A<B<int>, B<int>>;
+    // new-error@-1 {{ambiguous partial specializations}}
+  } // namespace t6
 } // namespace consistency
+
+namespace classes {
+  namespace canon {
+    template<class T, class U> struct A {};
+
+    template<template<class> class TT> auto f(TT<int> a) { return a; }
+    // old-note@-1 2{{template template argument has different template parameters}}
+    // new-note@-2 2{{substitution failure: too few template arguments}}
+
+    A<int, float> v1;
+    A<int, double> v2;
+
+    using X = decltype(f(v1));
+    // expected-error@-1 {{no matching function for call}}
+
+    using X = decltype(f(v2));
+    // expected-error@-1 {{no matching function for call}}
+  } // namespace canon
+  namespace expr {
+    template <class T1, int E1> struct A {
+      static constexpr auto val = E1;
+    };
+    template <template <class T3> class TT> void f(TT<int> v) {
+      // old-note@-1 {{template template argument has different template parameters}}
+      // new-note@-2 {{substitution failure: too few template arguments}}
+      static_assert(v.val == 3);
+    };
+    void test() {
+      f(A<int, 3>());
+      // expected-error@-1 {{no matching function for call}}
+    }
+  } // namespace expr
+  namespace packs {
+    template <class T1, class ...T2s> struct A {
+      static constexpr auto val = sizeof...(T2s);
+    };
+
+    template <template <class T3> class TT> void f(TT<int> v) {
+      // old-note@-1 {{template template argument has different template parameters}}
+      // new-note@-2 {{deduced type 'A<[...], (no argument), (no argument), (no argument)>' of 1st parameter does not match adjusted type 'A<[...], void, void, void>' of argument [with TT = A]}}
+      static_assert(v.val == 3);
+    };
+    void test() {
+      f(A<int, void, void, void>());
+      // expected-error@-1 {{no matching function for call}}
+    }
+  } // namespace packs
+} // namespace classes
+
+namespace regression1 {
+  template <typename T, typename Y> struct map {};
+  template <typename T> class foo {};
+
+  template <template <typename...> class MapType, typename Value>
+  Value bar(MapType<int, Value> map);
+
+  template <template <typename...> class MapType, typename Value>
+  Value bar(MapType<int, foo<Value>> map);
+
+  void aux() {
+    map<int, foo<int>> input;
+    bar(input);
+  }
+} // namespace regression1
diff --git a/clang/test/SemaTemplate/template-id-expr.cpp b/clang/test/SemaTemplate/template-id-expr.cpp
index 6c98e29cdaa9576b61f951faa1bdfbf901aaa628..dc12823ae307fb0edad124779604d69f3d4f54a2 100644
--- a/clang/test/SemaTemplate/template-id-expr.cpp
+++ b/clang/test/SemaTemplate/template-id-expr.cpp
@@ -179,7 +179,7 @@ template <template <typename> class D>
 class E {
   template class D<C>;  // expected-error {{expected '<' after 'template'}}
   template<> class D<C>;  // expected-error {{cannot specialize a template template parameter}}
-  friend class D<C>; // expected-error {{type alias template 'D' cannot be referenced with a class specifier}}
+  friend class D<C>; // expected-error {{alias template 'D' cannot be referenced with the 'class' specifier}}
 };
 #if __cplusplus <= 199711L
 // expected-warning@+2 {{extension}}
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 50dd073c41255f3749dfb516147d0b94f341c91f..6cba1267f3b0db50e5ca7d95962f0fd138f92aa5 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -509,8 +509,7 @@ static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) {
   if (OutputXML || DryRun) {
     if (DryRun)
       return emitReplacementWarnings(Replaces, AssumedFileName, Code);
-    else
-      outputXML(Replaces, FormatChanges, Status, Cursor, CursorPosition);
+    outputXML(Replaces, FormatChanges, Status, Cursor, CursorPosition);
   } else {
     IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
         new llvm::vfs::InMemoryFileSystem);
diff --git a/clang/unittests/AST/Interp/toAPValue.cpp b/clang/unittests/AST/Interp/toAPValue.cpp
index e56453aba2c5f725f918cd453df1961cb0935d6d..d6879d6e0bca36aed9d4f36ca427029538eacaee 100644
--- a/clang/unittests/AST/Interp/toAPValue.cpp
+++ b/clang/unittests/AST/Interp/toAPValue.cpp
@@ -186,3 +186,49 @@ TEST(ToAPValue, FunctionPointersC) {
     ASSERT_EQ(I, 17);
   }
 }
+
+TEST(ToAPValue, MemberPointers) {
+  constexpr char Code[] = "struct S {\n"
+                          "  int m, n;\n"
+                          "};\n"
+                          "constexpr int S::*pm = &S::m;\n"
+                          "constexpr int S::*nn = nullptr;\n";
+
+  auto AST = tooling::buildASTFromCodeWithArgs(
+      Code, {"-fexperimental-new-constant-interpreter"});
+
+  auto &Ctx = AST->getASTContext().getInterpContext();
+  Program &Prog = Ctx.getProgram();
+
+  auto getDecl = [&](const char *Name) -> const ValueDecl * {
+    auto Nodes =
+        match(valueDecl(hasName(Name)).bind("var"), AST->getASTContext());
+    assert(Nodes.size() == 1);
+    const auto *D = Nodes[0].getNodeAs<ValueDecl>("var");
+    assert(D);
+    return D;
+  };
+
+  auto getGlobalPtr = [&](const char *Name) -> Pointer {
+    const VarDecl *D = cast<VarDecl>(getDecl(Name));
+    return Prog.getPtrGlobal(*Prog.getGlobal(D));
+  };
+
+  {
+    const Pointer &GP = getGlobalPtr("pm");
+    ASSERT_TRUE(GP.isLive());
+    const MemberPointer &FP = GP.deref<MemberPointer>();
+    APValue A = FP.toAPValue();
+    ASSERT_EQ(A.getMemberPointerDecl(), getDecl("m"));
+    ASSERT_EQ(A.getKind(), APValue::MemberPointer);
+  }
+
+  {
+    const Pointer &GP = getGlobalPtr("nn");
+    ASSERT_TRUE(GP.isLive());
+    const MemberPointer &NP = GP.deref<MemberPointer>();
+    ASSERT_TRUE(NP.isZero());
+    APValue A = NP.toAPValue();
+    ASSERT_EQ(A.getKind(), APValue::MemberPointer);
+  }
+}
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 004ecb63f66205f0388aee99c96e3080df1fe747..dbc1916825f334a81f069525d2850e375084dd6d 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -9241,6 +9241,14 @@ TEST_F(FormatTest, AlignsAfterOpenBracket) {
                "        b));",
                Style);
 
+  Style.ColumnLimit = 30;
+  verifyFormat("for (int foo = 0; foo < FOO;\n"
+               "    ++foo) {\n"
+               "  bar(foo);\n"
+               "}",
+               Style);
+  Style.ColumnLimit = 80;
+
   Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
   Style.BinPackArguments = false;
   Style.BinPackParameters = false;
@@ -24879,7 +24887,7 @@ TEST_F(FormatTest, SkipMacroDefinitionBody) {
                Style);
 
   // With comments.
-  verifyFormat("/* */ #define A  a //  a  a", "/* */  # define A  a  //  a  a",
+  verifyFormat("/* */ #define A  a  //  a  a", "/* */  # define A  a  //  a  a",
                Style);
   verifyNoChange("/* */ #define A  a //  a  a", Style);
 
@@ -24891,6 +24899,15 @@ TEST_F(FormatTest, SkipMacroDefinitionBody) {
                "int aaa; // a",
                Style);
 
+  verifyNoChange(
+      "#define MACRO_WITH_COMMENTS()                                       \\\n"
+      "  public:                                                           \\\n"
+      "    /* Documentation parsed by Doxygen for the following method. */ \\\n"
+      "    static MyType getClassTypeId();                                 \\\n"
+      "    /** Normal comment for the following method. */                 \\\n"
+      "    virtual MyType getTypeId() const;",
+      Style);
+
   // multiline macro definitions
   verifyNoChange("#define A  a\\\n"
                  "  A  a \\\n "
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 3d609a1f041bc4bd0d0631e05d9530250fe039c9..8cc5c239d30a1c965e8d8a91941240976766950c 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -1496,12 +1496,82 @@ TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
 
 TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
   auto Tokens = annotate("__asm{\n"
-                         "a:\n"
-                         "};");
-  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+                         "\"a\":\n"
+                         ": x\n"
+                         ":};");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
-  EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace);
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace);
+
+  Tokens = annotate("__asm(\n"
+                    "\"a\":\n"
+                    ": x\n"
+                    ":);");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
+
+  Tokens = annotate("asm volatile (\n"
+                    "\"a_label:\"\n"
+                    ":\n"
+                    ": x\n"
+                    ":);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon);
+
+  Tokens = annotate("__asm__(\n"
+                    "\"a_label:\"\n"
+                    ": x\n"
+                    ":\n"
+                    ": y);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
+
+  Tokens = annotate("__asm volatile (\n"
+                    "\"a_label:\"\n"
+                    "\"a b c(%%x)\"\n"
+                    ":\n"
+                    ": x\n"
+                    ":);");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon);
+
+  Tokens = annotate("asm(\n"
+                    "\"insn\"\n"
+                    ": \"=r\" (var1), \"=&r\" (value)\n"
+                    ":\n"
+                    ": \"memory\");");
+  ASSERT_EQ(Tokens.size(), 19u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon);
+
+  Tokens = annotate("__asm__ volatile (\n"
+                    "\"ldr r1, [r0, %%[sym]]\"\n"
+                    ":\n"
+                    ": [sym] \"J\" (aaaaa(aaaa, aaaa))\n"
+                    ");");
+  ASSERT_EQ(Tokens.size(), 21u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
+  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
+  EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp
index ab9b7a31094f7105ebc7492e792da73d16c20ac6..72b34dae378e505f4f8150c77baba7d30c996171 100644
--- a/clang/unittests/Interpreter/InterpreterTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterTest.cpp
@@ -282,6 +282,9 @@ TEST_F(InterpreterTest, InstantiateTemplate) {
   EXPECT_EQ(42, fn(NewA.getPtr()));
 }
 
+// This test exposes an ARM specific problem in the interpreter, see
+// https://github.com/llvm/llvm-project/issues/94741.
+#ifndef __arm__
 TEST_F(InterpreterTest, Value) {
   std::unique_ptr<Interpreter> Interp = createInterpreter();
 
@@ -379,5 +382,6 @@ TEST_F(InterpreterTest, Value) {
   EXPECT_EQ(V9.getKind(), Value::K_PtrOrObj);
   EXPECT_TRUE(V9.isManuallyAlloc());
 }
+#endif /* ifndef __arm__ */
 
 } // end anonymous namespace
diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index 044c3d65ec6fbf23b097e5d26401e1b0337d21d4..59fef9ecbb9c916106bd6fbb278d634121c57c0b 100644
--- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/DependencyDirectivesScanner.h"
-#include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/SmallString.h"
 #include "gtest/gtest.h"
 
@@ -18,11 +17,11 @@ using namespace clang::dependency_directives_scan;
 static bool minimizeSourceToDependencyDirectives(
     StringRef Input, SmallVectorImpl<char> &Out,
     SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-    SmallVectorImpl<Directive> &Directives, const LangOptions &LangOpts) {
+    SmallVectorImpl<Directive> &Directives) {
   Out.clear();
   Tokens.clear();
   Directives.clear();
-  if (scanSourceForDependencyDirectives(Input, Tokens, Directives, LangOpts))
+  if (scanSourceForDependencyDirectives(Input, Tokens, Directives))
     return true;
 
   raw_svector_ostream OS(Out);
@@ -39,9 +38,7 @@ static bool minimizeSourceToDependencyDirectives(StringRef Input,
                                                  SmallVectorImpl<char> &Out) {
   SmallVector<dependency_directives_scan::Token, 16> Tokens;
   SmallVector<Directive, 32> Directives;
-  LangOptions LangOpts;
-  return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives,
-                                              LangOpts);
+  return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives);
 }
 
 namespace {
@@ -50,17 +47,16 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Empty) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
 
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives("", Out, Tokens, Directives,
-                                                    LangOpts));
+  ASSERT_FALSE(
+      minimizeSourceToDependencyDirectives("", Out, Tokens, Directives));
   EXPECT_TRUE(Out.empty());
   EXPECT_TRUE(Tokens.empty());
   ASSERT_EQ(1u, Directives.size());
   ASSERT_EQ(pp_eof, Directives.back().Kind);
 
   ASSERT_FALSE(minimizeSourceToDependencyDirectives("abc def\nxyz", Out, Tokens,
-                                                    Directives, LangOpts));
+                                                    Directives));
   EXPECT_STREQ("<TokBeforeEOF>\n", Out.data());
   EXPECT_TRUE(Tokens.empty());
   ASSERT_EQ(2u, Directives.size());
@@ -72,7 +68,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
 
   ASSERT_FALSE(
       minimizeSourceToDependencyDirectives("#define A\n"
@@ -97,7 +92,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
                                            "export module m;\n"
                                            "import m;\n"
                                            "#pragma clang system_header\n",
-                                           Out, Tokens, Directives, LangOpts));
+                                           Out, Tokens, Directives));
   EXPECT_EQ(pp_define, Directives[0].Kind);
   EXPECT_EQ(pp_undef, Directives[1].Kind);
   EXPECT_EQ(pp_endif, Directives[2].Kind);
@@ -150,10 +145,9 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Define) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
 
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives(
-      "#define MACRO", Out, Tokens, Directives, LangOpts));
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("#define MACRO", Out,
+                                                    Tokens, Directives));
   EXPECT_STREQ("#define MACRO\n", Out.data());
   ASSERT_EQ(4u, Tokens.size());
   ASSERT_EQ(2u, Directives.size());
@@ -844,7 +838,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
 
   StringRef Source = R"(// comment
 #pragma once
@@ -852,8 +845,8 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) {
 #include <test.h>
 _Pragma("once")
 )";
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens,
-                                                    Directives, LangOpts));
+  ASSERT_FALSE(
+      minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives));
   EXPECT_STREQ("#pragma once\n#include <test.h>\n_Pragma(\"once\")\n",
                Out.data());
   ASSERT_EQ(Directives.size(), 4u);
@@ -933,7 +926,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, CxxModules) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
 
   StringRef Source = R"(
     module;
@@ -962,8 +954,8 @@ ort \
       import f(->a = 3);
     }
     )";
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens,
-                                                    Directives, LangOpts));
+  ASSERT_FALSE(
+      minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives));
   EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;"
                "exp\\\nort import:l[[rename]];"
                "import<<=3;import a b d e d e f e;"
@@ -1020,52 +1012,4 @@ TEST(MinimizeSourceToDependencyDirectivesTest, TokensBeforeEOF) {
   EXPECT_STREQ("#ifndef A\n#define A\n#endif\n<TokBeforeEOF>\n", Out.data());
 }
 
-TEST(MinimizeSourceToDependencyDirectivesTest, CPlusPlus14PPNumber) {
-  SmallVector<char, 128> Out;
-  SmallVector<dependency_directives_scan::Token, 4> Tokens;
-  SmallVector<Directive, 4> Directives;
-  LangOptions LangOpts;
-
-  StringRef Source = R"(
-#if 123'124
-#endif
-)";
-
-  LangOpts.CPlusPlus14 = true;
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens,
-                                                    Directives, LangOpts));
-  EXPECT_STREQ("#if 123'124\n#endif\n", Out.data());
-  ASSERT_EQ(Directives.size(), 3u);
-  EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_if);
-  EXPECT_EQ(Directives[1].Kind, dependency_directives_scan::pp_endif);
-  EXPECT_EQ(Directives[2].Kind, dependency_directives_scan::pp_eof);
-  ASSERT_EQ(Tokens.size(), 7u);
-
-  ASSERT_TRUE(Tokens[0].is(tok::hash));
-  ASSERT_TRUE(Tokens[1].is(tok::raw_identifier));   // "if"
-  ASSERT_TRUE(Tokens[2].is(tok::numeric_constant)); // 123'124
-  ASSERT_TRUE(Tokens[3].is(tok::eod));
-  ASSERT_TRUE(Tokens[4].is(tok::hash));
-  ASSERT_TRUE(Tokens[5].is(tok::raw_identifier)); // #endif
-  ASSERT_TRUE(Tokens[6].is(tok::eod));
-
-  LangOpts.CPlusPlus14 = false;
-  ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens,
-                                                    Directives, LangOpts));
-  EXPECT_STREQ("#if 123'124\n#endif\n", Out.data());
-  ASSERT_EQ(Directives.size(), 3u);
-  EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_if);
-  EXPECT_EQ(Directives[1].Kind, dependency_directives_scan::pp_endif);
-  EXPECT_EQ(Directives[2].Kind, dependency_directives_scan::pp_eof);
-  ASSERT_EQ(Tokens.size(), 8u);
-  ASSERT_TRUE(Tokens[0].is(tok::hash));
-  ASSERT_TRUE(Tokens[1].is(tok::raw_identifier));   // "if"
-  ASSERT_TRUE(Tokens[2].is(tok::numeric_constant)); // 123
-  ASSERT_TRUE(Tokens[3].is(tok::unknown));          // '124
-  ASSERT_TRUE(Tokens[4].is(tok::eod));
-  ASSERT_TRUE(Tokens[5].is(tok::hash));
-  ASSERT_TRUE(Tokens[6].is(tok::raw_identifier)); // #endif
-  ASSERT_TRUE(Tokens[7].is(tok::eod));
-}
-
 } // end anonymous namespace
diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp
index 410f378f1e89d466daa82f137dc0c1bcede9cb59..6ff87f720a559fd6276fd79eb3a2414efbed5cda 100644
--- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp
+++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp
@@ -104,7 +104,6 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) {
     SmallVector<dependency_directives_scan::Directive> Directives;
   };
   SmallVector<std::unique_ptr<DepDirectives>> DepDirectivesObjects;
-  LangOptions LangOpts;
 
   auto getDependencyDirectives = [&](FileEntryRef File)
       -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
@@ -112,7 +111,7 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) {
     StringRef Input = (*FileMgr.getBufferForFile(File))->getBuffer();
     bool Err = scanSourceForDependencyDirectives(
         Input, DepDirectivesObjects.back()->Tokens,
-        DepDirectivesObjects.back()->Directives, LangOpts);
+        DepDirectivesObjects.back()->Directives);
     EXPECT_FALSE(Err);
     return llvm::ArrayRef(DepDirectivesObjects.back()->Directives);
   };
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 42f51697139658dc5bb8807c9017ce2aa674e03b..dc8a11dbc345ca4f0b5b5062882824079eb27d2d 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -374,11 +374,24 @@ TEST_F(TokenCollectorTest, Locations) {
 
   auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
   for (auto &R : Code.ranges()) {
-    EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)),
-                Pointee(RangeIs(R)));
+    EXPECT_THAT(
+        Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(R.Begin)),
+        Pointee(RangeIs(R)));
   }
 }
 
+TEST_F(TokenCollectorTest, LocationInMiddleOfSpelledToken) {
+  llvm::Annotations Code(R"cpp(
+    int foo = [[baa^aar]];
+  )cpp");
+  recordTokens(Code.code());
+  // Check spelled tokens.
+  auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
+  EXPECT_THAT(
+      Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(Code.point())),
+      Pointee(RangeIs(Code.range())));
+}
+
 TEST_F(TokenCollectorTest, MacroDirectives) {
   // Macro directives are not stored anywhere at the moment.
   std::string Code = R"cpp(
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index b046468c853167dfe67fb0cac529faa9f512d9fe..5e2ab06701703b6a0b1a4c68e5e334cabdeb51d2 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -10698,7 +10698,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1815.html">1815</a></td>
     <td>CD4</td>
     <td>Lifetime extension in aggregate initialization</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="none" align="center">No</td>
   </tr>
   <tr id="1816">
     <td><a href="https://cplusplus.github.io/CWG/issues/1816.html">1816</a></td>
@@ -17186,6 +17186,18 @@ objects</td>
     <td>open</td>
     <td>Template argument deduction involving exception specifications</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2897">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2897.html">2897</a></td>
+    <td>open</td>
+    <td>Copying potentially-overlapping union subobjects</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2898">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2898.html">2898</a></td>
+    <td>open</td>
+    <td>Clarify implicit conversion sequence from <I>cv</I> <TT>T</TT> to <TT>T</TT></td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>
diff --git a/compiler-rt/lib/builtins/atomic.c b/compiler-rt/lib/builtins/atomic.c
index c3a36a9aaba60764c8274d18231b20460624d794..852bb20f086723b896c5a09867aace6a46b5c665 100644
--- a/compiler-rt/lib/builtins/atomic.c
+++ b/compiler-rt/lib/builtins/atomic.c
@@ -51,14 +51,6 @@
 #endif
 static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1;
 
-#ifndef CACHE_LINE_SIZE
-#define CACHE_LINE_SIZE 64
-#endif
-
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wgnu-designator"
-#endif
-
 ////////////////////////////////////////////////////////////////////////////////
 // Platform-specific lock implementation.  Falls back to spinlocks if none is
 // defined.  Each platform should define the Lock type, and corresponding
@@ -102,18 +94,21 @@ static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0
 #else
 _Static_assert(__atomic_always_lock_free(sizeof(uintptr_t), 0),
                "Implementation assumes lock-free pointer-size cmpxchg");
-#include <pthread.h>
-#include <stdalign.h>
-typedef struct {
-  alignas(CACHE_LINE_SIZE) pthread_mutex_t m;
-} Lock;
+typedef _Atomic(uintptr_t) Lock;
 /// Unlock a lock.  This is a release operation.
-__inline static void unlock(Lock *l) { pthread_mutex_unlock(&l->m); }
-/// Locks a lock.
-__inline static void lock(Lock *l) { pthread_mutex_lock(&l->m); }
+__inline static void unlock(Lock *l) {
+  __c11_atomic_store(l, 0, __ATOMIC_RELEASE);
+}
+/// Locks a lock.  In the current implementation, this is potentially
+/// unbounded in the contended case.
+__inline static void lock(Lock *l) {
+  uintptr_t old = 0;
+  while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE,
+                                             __ATOMIC_RELAXED))
+    old = 0;
+}
 /// locks for atomic operations
-static Lock locks[SPINLOCK_COUNT] = {
-    [0 ... SPINLOCK_COUNT - 1] = {PTHREAD_MUTEX_INITIALIZER}};
+static Lock locks[SPINLOCK_COUNT];
 #endif
 
 /// Returns a lock to use for a given pointer.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h b/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
index 07a59ab11c4293a785d1e0df138cc6a50d113bb2..eef1e7e9d95702a3b5db07d27f92d6760a56c414 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
@@ -321,23 +321,23 @@ class TwoLevelBitVector {
   };
 
  private:
-  void check(uptr idx) const { CHECK_LE(idx, size()); }
+  void check(uptr idx) const { CHECK_LT(idx, size()); }
 
   uptr idx0(uptr idx) const {
     uptr res = idx / (BV::kSize * BV::kSize);
-    CHECK_LE(res, kLevel1Size);
+    CHECK_LT(res, kLevel1Size);
     return res;
   }
 
   uptr idx1(uptr idx) const {
     uptr res = (idx / BV::kSize) % BV::kSize;
-    CHECK_LE(res, BV::kSize);
+    CHECK_LT(res, BV::kSize);
     return res;
   }
 
   uptr idx2(uptr idx) const {
     uptr res = idx % BV::kSize;
-    CHECK_LE(res, BV::kSize);
+    CHECK_LT(res, BV::kSize);
     return res;
   }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index cccbb4d256df2f64d6fc6f0f8906f2e35625bc78..6d05411222d9e831562660dcb186563be8aced32 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -54,6 +54,8 @@
 #    undef MAP_NORESERVE
 #    define MAP_NORESERVE 0
 extern const Elf_Auxinfo *__elf_aux_vector;
+extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
+                               struct sigaction *oldact);
 #  endif
 
 #  if SANITIZER_NETBSD
@@ -93,12 +95,22 @@ SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,
                                             void *oldact);
 
 int internal_sigaction(int signum, const void *act, void *oldact) {
-#  if !SANITIZER_GO
+#  if SANITIZER_FREEBSD
+  // On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD
+  // 15) since the libc version goes via a global interposing table. Due to
+  // library initialization order the table can be relocated after the call to
+  // InitializeDeadlySignals() which then crashes when dereferencing the
+  // uninitialized pointer in libc.
+  return __sys_sigaction(signum, (const struct sigaction *)act,
+                         (struct sigaction *)oldact);
+#  else
+#    if !SANITIZER_GO
   if (&real_sigaction)
     return real_sigaction(signum, act, oldact);
-#  endif
+#    endif
   return sigaction(signum, (const struct sigaction *)act,
                    (struct sigaction *)oldact);
+#  endif
 }
 
 void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
index 58f2c8f7b3334f289e5e8e5536415b72ae55caf1..1a1ccce82d259b85d3cd8fae2347730a611c29b8 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
@@ -69,12 +69,17 @@ const uptr kAllocatorSpace = ~(uptr)0;
 const uptr kAllocatorSize = 0x2000000000ULL;  // 128G.
 static const u64 kAddressSpaceSize = 1ULL << 38;
 typedef VeryDenseSizeClassMap SizeClassMap;
-#else
+#    elif SANITIZER_APPLE
 static const uptr kAllocatorSpace = 0x700000000000ULL;
 static const uptr kAllocatorSize  = 0x010000000000ULL;  // 1T.
 static const u64 kAddressSpaceSize = 1ULL << 47;
 typedef DefaultSizeClassMap SizeClassMap;
-#endif
+#    else
+static const uptr kAllocatorSpace = 0x500000000000ULL;
+static const uptr kAllocatorSize = 0x010000000000ULL;  // 1T.
+static const u64 kAddressSpaceSize = 1ULL << 47;
+typedef DefaultSizeClassMap SizeClassMap;
+#    endif
 
 template <typename AddressSpaceViewTy>
 struct AP64 {  // Allocator Params. Short name for shorter demangled names..
diff --git a/compiler-rt/lib/xray/tests/unit/function_call_trie_test.cpp b/compiler-rt/lib/xray/tests/unit/function_call_trie_test.cpp
index c90d6637fff5e5074b0621d18e15a34e80119f16..b058e3ced2792b2440574536b808bb2cead23481 100644
--- a/compiler-rt/lib/xray/tests/unit/function_call_trie_test.cpp
+++ b/compiler-rt/lib/xray/tests/unit/function_call_trie_test.cpp
@@ -310,16 +310,14 @@ TEST(FunctionCallTrieTest, MergeInto) {
 
 TEST(FunctionCallTrieTest, PlacementNewOnAlignedStorage) {
   profilingFlags()->setDefaults();
-  typename std::aligned_storage<sizeof(FunctionCallTrie::Allocators),
-                                alignof(FunctionCallTrie::Allocators)>::type
-      AllocatorsStorage;
+  alignas(FunctionCallTrie::Allocators)
+      std::byte AllocatorsStorage[sizeof(FunctionCallTrie::Allocators)];
   new (&AllocatorsStorage)
       FunctionCallTrie::Allocators(FunctionCallTrie::InitAllocators());
   auto *A =
       reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage);
 
-  typename std::aligned_storage<sizeof(FunctionCallTrie),
-                                alignof(FunctionCallTrie)>::type FCTStorage;
+  alignas(FunctionCallTrie) std::byte FCTStorage[sizeof(FunctionCallTrie)];
   new (&FCTStorage) FunctionCallTrie(*A);
   auto *T = reinterpret_cast<FunctionCallTrie *>(&FCTStorage);
 
diff --git a/compiler-rt/lib/xray/tests/unit/profile_collector_test.cpp b/compiler-rt/lib/xray/tests/unit/profile_collector_test.cpp
index eab5579cb3ee0574f3ecbfc60e43d792acca888b..da50642284d435d78be42f17ee3f6ebe182d6b6e 100644
--- a/compiler-rt/lib/xray/tests/unit/profile_collector_test.cpp
+++ b/compiler-rt/lib/xray/tests/unit/profile_collector_test.cpp
@@ -38,8 +38,8 @@ struct ExpectedProfilingFileHeader {
 void ValidateFileHeaderBlock(XRayBuffer B) {
   ASSERT_NE(static_cast<const void *>(B.Data), nullptr);
   ASSERT_EQ(B.Size, sizeof(ExpectedProfilingFileHeader));
-  typename std::aligned_storage<sizeof(ExpectedProfilingFileHeader)>::type
-      FileHeaderStorage;
+  alignas(ExpectedProfilingFileHeader)
+      std::byte FileHeaderStorage[sizeof(ExpectedProfilingFileHeader)];
   ExpectedProfilingFileHeader ExpectedHeader;
   std::memcpy(&FileHeaderStorage, B.Data, B.Size);
   auto &FileHeader =
diff --git a/compiler-rt/lib/xray/tests/unit/segmented_array_test.cpp b/compiler-rt/lib/xray/tests/unit/segmented_array_test.cpp
index 46aeb88f71b4cd39ae446830550737b617142742..26c80ded1d7ecc659de8e1df801a2d51ae14de1d 100644
--- a/compiler-rt/lib/xray/tests/unit/segmented_array_test.cpp
+++ b/compiler-rt/lib/xray/tests/unit/segmented_array_test.cpp
@@ -226,13 +226,11 @@ TEST(SegmentedArrayTest, SimulateStackBehaviour) {
 
 TEST(SegmentedArrayTest, PlacementNewOnAlignedStorage) {
   using AllocatorType = typename Array<ShadowStackEntry>::AllocatorType;
-  typename std::aligned_storage<sizeof(AllocatorType),
-                                alignof(AllocatorType)>::type AllocatorStorage;
+  alignas(AllocatorType) std::byte AllocatorStorage[sizeof(AllocatorType)];
   new (&AllocatorStorage) AllocatorType(1 << 10);
   auto *A = reinterpret_cast<AllocatorType *>(&AllocatorStorage);
-  typename std::aligned_storage<sizeof(Array<ShadowStackEntry>),
-                                alignof(Array<ShadowStackEntry>)>::type
-      ArrayStorage;
+  alignas(Array<ShadowStackEntry>)
+      std::byte ArrayStorage[sizeof(Array<ShadowStackEntry>)];
   new (&ArrayStorage) Array<ShadowStackEntry>(*A);
   auto *Data = reinterpret_cast<Array<ShadowStackEntry> *>(&ArrayStorage);
 
diff --git a/compiler-rt/lib/xray/tests/unit/test_helpers.cpp b/compiler-rt/lib/xray/tests/unit/test_helpers.cpp
index 6075f36a4a78a80385fb259792becd113940f45d..81a93d8720a6c29c2332c1b11658f21a672607a1 100644
--- a/compiler-rt/lib/xray/tests/unit/test_helpers.cpp
+++ b/compiler-rt/lib/xray/tests/unit/test_helpers.cpp
@@ -69,8 +69,7 @@ namespace __xray {
 
 std::string serialize(BufferQueue &Buffers, int32_t Version) {
   std::string Serialized;
-  std::aligned_storage<sizeof(XRayFileHeader), alignof(XRayFileHeader)>::type
-      HeaderStorage;
+  alignas(XRayFileHeader) std::byte HeaderStorage[sizeof(XRayFileHeader)];
   auto *Header = reinterpret_cast<XRayFileHeader *>(&HeaderStorage);
   new (Header) XRayFileHeader();
   Header->Version = Version;
diff --git a/compiler-rt/lib/xray/xray_fdr_logging.cpp b/compiler-rt/lib/xray/xray_fdr_logging.cpp
index 378a8c0f4a70a0205204e68920772a066b5db681..7def3565d56ab8e2c766712f2782a5f3f49e21bb 100644
--- a/compiler-rt/lib/xray/xray_fdr_logging.cpp
+++ b/compiler-rt/lib/xray/xray_fdr_logging.cpp
@@ -55,17 +55,12 @@ struct XRAY_TLS_ALIGNAS(64) ThreadLocalData {
   BufferQueue::Buffer Buffer{};
   BufferQueue *BQ = nullptr;
 
-  using LogWriterStorage =
-      typename std::aligned_storage<sizeof(FDRLogWriter),
-                                    alignof(FDRLogWriter)>::type;
-
-  LogWriterStorage LWStorage;
+  using LogWriterStorage = std::byte[sizeof(FDRLogWriter)];
+  alignas(FDRLogWriter) LogWriterStorage LWStorage;
   FDRLogWriter *Writer = nullptr;
 
-  using ControllerStorage =
-      typename std::aligned_storage<sizeof(FDRController<>),
-                                    alignof(FDRController<>)>::type;
-  ControllerStorage CStorage;
+  using ControllerStorage = std::byte[sizeof(FDRController<>)];
+  alignas(FDRController<>) ControllerStorage CStorage;
   FDRController<> *Controller = nullptr;
 };
 
@@ -78,7 +73,7 @@ static_assert(std::is_trivially_destructible<ThreadLocalData>::value,
 static pthread_key_t Key;
 
 // Global BufferQueue.
-static std::aligned_storage<sizeof(BufferQueue)>::type BufferQueueStorage;
+static std::byte BufferQueueStorage[sizeof(BufferQueue)];
 static BufferQueue *BQ = nullptr;
 
 // Global thresholds for function durations.
@@ -129,8 +124,8 @@ static_assert(alignof(ThreadLocalData) >= 64,
               "ThreadLocalData must be cache line aligned.");
 #endif
 static ThreadLocalData &getThreadLocalData() {
-  thread_local typename std::aligned_storage<
-      sizeof(ThreadLocalData), alignof(ThreadLocalData)>::type TLDStorage{};
+  alignas(ThreadLocalData) thread_local std::byte
+      TLDStorage[sizeof(ThreadLocalData)];
 
   if (pthread_getspecific(Key) == NULL) {
     new (reinterpret_cast<ThreadLocalData *>(&TLDStorage)) ThreadLocalData{};
diff --git a/compiler-rt/lib/xray/xray_function_call_trie.h b/compiler-rt/lib/xray/xray_function_call_trie.h
index b8c60583761b547c8ef07ea2453d53d5b6c715a4..7536f39b8081aeb868c340203678041c8e3ec6ed 100644
--- a/compiler-rt/lib/xray/xray_function_call_trie.h
+++ b/compiler-rt/lib/xray/xray_function_call_trie.h
@@ -139,18 +139,14 @@ public:
 
     // Use hosted aligned storage members to allow for trivial move and init.
     // This also allows us to sidestep the potential-failing allocation issue.
-    typename std::aligned_storage<sizeof(NodeAllocatorType),
-                                  alignof(NodeAllocatorType)>::type
-        NodeAllocatorStorage;
-    typename std::aligned_storage<sizeof(RootAllocatorType),
-                                  alignof(RootAllocatorType)>::type
-        RootAllocatorStorage;
-    typename std::aligned_storage<sizeof(ShadowStackAllocatorType),
-                                  alignof(ShadowStackAllocatorType)>::type
-        ShadowStackAllocatorStorage;
-    typename std::aligned_storage<sizeof(NodeIdPairAllocatorType),
-                                  alignof(NodeIdPairAllocatorType)>::type
-        NodeIdPairAllocatorStorage;
+    alignas(NodeAllocatorType) std::byte
+        NodeAllocatorStorage[sizeof(NodeAllocatorType)];
+    alignas(RootAllocatorType) std::byte
+        RootAllocatorStorage[sizeof(RootAllocatorType)];
+    alignas(ShadowStackAllocatorType) std::byte
+        ShadowStackAllocatorStorage[sizeof(ShadowStackAllocatorType)];
+    alignas(NodeIdPairAllocatorType) std::byte
+        NodeIdPairAllocatorStorage[sizeof(NodeIdPairAllocatorType)];
 
     NodeAllocatorType *NodeAllocator = nullptr;
     RootAllocatorType *RootAllocator = nullptr;
diff --git a/compiler-rt/lib/xray/xray_profile_collector.cpp b/compiler-rt/lib/xray/xray_profile_collector.cpp
index bef2504f2a16ec65cfb7d0e17076df4d37af400c..3a28240e603c990e147089d2b19d9e68fa763538 100644
--- a/compiler-rt/lib/xray/xray_profile_collector.cpp
+++ b/compiler-rt/lib/xray/xray_profile_collector.cpp
@@ -29,7 +29,7 @@ namespace {
 SpinMutex GlobalMutex;
 struct ThreadTrie {
   tid_t TId;
-  typename std::aligned_storage<sizeof(FunctionCallTrie)>::type TrieStorage;
+  alignas(FunctionCallTrie) std::byte TrieStorage[sizeof(FunctionCallTrie)];
 };
 
 struct ProfileBuffer {
@@ -71,16 +71,13 @@ using ThreadDataAllocator = ThreadDataArray::AllocatorType;
 // by the ThreadData array. This lets us host the buffers, allocators, and tries
 // associated with a thread by moving the data into the array instead of
 // attempting to copy the data to a separately backed set of tries.
-static typename std::aligned_storage<
-    sizeof(BufferQueue), alignof(BufferQueue)>::type BufferQueueStorage;
+alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)];
 static BufferQueue *BQ = nullptr;
 static BufferQueue::Buffer Buffer;
-static typename std::aligned_storage<sizeof(ThreadDataAllocator),
-                                     alignof(ThreadDataAllocator)>::type
-    ThreadDataAllocatorStorage;
-static typename std::aligned_storage<sizeof(ThreadDataArray),
-                                     alignof(ThreadDataArray)>::type
-    ThreadDataArrayStorage;
+alignas(ThreadDataAllocator) static std::byte
+    ThreadDataAllocatorStorage[sizeof(ThreadDataAllocator)];
+alignas(ThreadDataArray) static std::byte
+    ThreadDataArrayStorage[sizeof(ThreadDataArray)];
 
 static ThreadDataAllocator *TDAllocator = nullptr;
 static ThreadDataArray *TDArray = nullptr;
@@ -91,10 +88,10 @@ using ProfileBufferArrayAllocator = typename ProfileBufferArray::AllocatorType;
 // These need to be global aligned storage to avoid dynamic initialization. We
 // need these to be aligned to allow us to placement new objects into the
 // storage, and have pointers to those objects be appropriately aligned.
-static typename std::aligned_storage<sizeof(ProfileBufferArray)>::type
-    ProfileBuffersStorage;
-static typename std::aligned_storage<sizeof(ProfileBufferArrayAllocator)>::type
-    ProfileBufferArrayAllocatorStorage;
+alignas(ProfileBufferArray) static std::byte
+    ProfileBuffersStorage[sizeof(ProfileBufferArray)];
+alignas(ProfileBufferArrayAllocator) static std::byte
+    ProfileBufferArrayAllocatorStorage[sizeof(ProfileBufferArrayAllocator)];
 
 static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr;
 static ProfileBufferArray *ProfileBuffers = nullptr;
@@ -382,8 +379,8 @@ XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT {
     return {nullptr, 0};
 
   static pthread_once_t Once = PTHREAD_ONCE_INIT;
-  static typename std::aligned_storage<sizeof(XRayProfilingFileHeader)>::type
-      FileHeaderStorage;
+  alignas(XRayProfilingFileHeader) static std::byte
+      FileHeaderStorage[sizeof(XRayProfilingFileHeader)];
   pthread_once(
       &Once, +[]() XRAY_NEVER_INSTRUMENT {
         new (&FileHeaderStorage) XRayProfilingFileHeader{};
diff --git a/compiler-rt/lib/xray/xray_profiling.cpp b/compiler-rt/lib/xray/xray_profiling.cpp
index 259ec65a76a1f632d184ebb7e0e731e9a2bdb3e9..e9ac2fdd8aadf7bb3b07dcb6d5f198b63c4d05cd 100644
--- a/compiler-rt/lib/xray/xray_profiling.cpp
+++ b/compiler-rt/lib/xray/xray_profiling.cpp
@@ -48,17 +48,14 @@ static pthread_key_t ProfilingKey;
 
 // We use a global buffer queue, which gets initialized once at initialisation
 // time, and gets reset when profiling is "done".
-static std::aligned_storage<sizeof(BufferQueue), alignof(BufferQueue)>::type
-    BufferQueueStorage;
+alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)];
 static BufferQueue *BQ = nullptr;
 
 thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers;
-thread_local std::aligned_storage<sizeof(FunctionCallTrie::Allocators),
-                                  alignof(FunctionCallTrie::Allocators)>::type
-    AllocatorsStorage;
-thread_local std::aligned_storage<sizeof(FunctionCallTrie),
-                                  alignof(FunctionCallTrie)>::type
-    FunctionCallTrieStorage;
+alignas(FunctionCallTrie::Allocators) thread_local std::byte
+    AllocatorsStorage[sizeof(FunctionCallTrie::Allocators)];
+alignas(FunctionCallTrie) thread_local std::byte
+    FunctionCallTrieStorage[sizeof(FunctionCallTrie)];
 thread_local ProfilingData TLD{{0}, {0}};
 thread_local atomic_uint8_t ReentranceGuard{0};
 
diff --git a/compiler-rt/lib/xray/xray_segmented_array.h b/compiler-rt/lib/xray/xray_segmented_array.h
index 6eb673edffea47ffc35740a7f7bb03d608b5a059..3ab174bcbe18c4535e10a85543ed2ba6e7216d78 100644
--- a/compiler-rt/lib/xray/xray_segmented_array.h
+++ b/compiler-rt/lib/xray/xray_segmented_array.h
@@ -56,8 +56,7 @@ public:
   //     kCacheLineSize-multiple segments, minus the size of two pointers.
   //
   //   - Request cacheline-multiple sized elements from the allocator.
-  static constexpr uint64_t AlignedElementStorageSize =
-      sizeof(typename std::aligned_storage<sizeof(T), alignof(T)>::type);
+  static constexpr uint64_t AlignedElementStorageSize = sizeof(T);
 
   static constexpr uint64_t SegmentControlBlockSize = sizeof(Segment *) * 2;
 
diff --git a/compiler-rt/test/dfsan/release_shadow_space.c b/compiler-rt/test/dfsan/release_shadow_space.c
index 60dec98ebec49edffacc07b430faccf68563cf56..0f0e1a9c731cb4b0e9403c32fbf74e136152c0a5 100644
--- a/compiler-rt/test/dfsan/release_shadow_space.c
+++ b/compiler-rt/test/dfsan/release_shadow_space.c
@@ -3,9 +3,6 @@
 // DFSAN_OPTIONS=no_huge_pages_for_shadow=false RUN: %clang_dfsan %s -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 -o %t && %run %t
 // DFSAN_OPTIONS=no_huge_pages_for_shadow=true RUN: %clang_dfsan %s -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 -o %t && %run %t
 
-// This test is flaky right now: https://github.com/llvm/llvm-project/issues/91287
-// UNSUPPORTED:  target={{.*}}
-
 #include <assert.h>
 #include <sanitizer/dfsan_interface.h>
 #include <stdbool.h>
@@ -26,7 +23,11 @@ size_t get_rss_kb() {
   char buf[256];
   while (fgets(buf, sizeof(buf), f) != NULL) {
     int64_t rss;
-    if (sscanf(buf, "Rss: %ld kB", &rss) == 1)
+    // DFSan's sscanf is broken and doesn't check for ordinary characters in
+    // the format string, hence we use strstr as a secondary check
+    // (https://github.com/llvm/llvm-project/issues/94769).
+    if ((sscanf(buf, "Rss: %ld kB", &rss) == 1) &&
+        (strstr(buf, "Rss: ") != NULL))
       ret += rss;
   }
   assert(feof(f));
@@ -73,6 +74,11 @@ int main(int argc, char **argv) {
       before, after_mmap, after_mmap_and_set_label, after_fixed_mmap,
       after_mmap_and_set_label2, after_munmap);
 
+  // This is orders of magnitude larger than we expect (typically < 10,000KB).
+  // It is a quick check to ensure that the RSS calculation function isn't
+  // egregriously wrong.
+  assert(before < 1000000);
+
   const size_t mmap_cost_kb = map_size >> 10;
   // Shadow space (1:1 with application memory)
   const size_t mmap_shadow_cost_kb = sizeof(dfsan_label) * mmap_cost_kb;
diff --git a/compiler-rt/test/dfsan/sscanf.c b/compiler-rt/test/dfsan/sscanf.c
new file mode 100644
index 0000000000000000000000000000000000000000..dbc2de4ba96c1dbd557aa44fd05ebb2c31ade9ef
--- /dev/null
+++ b/compiler-rt/test/dfsan/sscanf.c
@@ -0,0 +1,19 @@
+// RUN: %clang_dfsan %s -o %t && %run %t
+// XFAIL: *
+
+#include <assert.h>
+#include <stdio.h>
+
+int main(int argc, char *argv[]) {
+  char buf[256] = "10000000000-100000000000 rw-p 00000000 00:00 0";
+  long rss = 0;
+  // This test exposes a bug in DFSan's sscanf, that leads to flakiness
+  // in release_shadow_space.c (see
+  // https://github.com/llvm/llvm-project/issues/91287)
+  if (sscanf(buf, "Garbage text before, %ld, Garbage text after", &rss) == 1) {
+    printf("Error: matched %ld\n", rss);
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index fae1d1686e569159f7a467eb8c2f2c5af3b0b00e..64b9a9d025e7b53c3fc641134511135abe567411 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -744,6 +744,13 @@ def is_binutils_lto_supported():
     return True
 
 
+def is_lld_lto_supported():
+    # LLD does support LTO, but we require it to be built with the latest
+    # changes to claim support. Otherwise older copies of LLD may not
+    # understand new bitcode versions.
+    return os.path.exists(os.path.join(config.llvm_tools_dir, "lld"))
+
+
 def is_windows_lto_supported():
     if not target_is_msvc:
         return True
@@ -755,7 +762,7 @@ if config.host_os == "Darwin" and is_darwin_lto_supported():
     config.lto_flags = ["-Wl,-lto_library," + liblto_path()]
 elif config.host_os in ["Linux", "FreeBSD", "NetBSD"]:
     config.lto_supported = False
-    if config.use_lld:
+    if config.use_lld and is_lld_lto_supported():
         config.lto_supported = True
     if is_binutils_lto_supported():
         config.available_features.add("binutils_lto")
diff --git a/compiler-rt/test/tsan/custom_mutex4.cpp b/compiler-rt/test/tsan/custom_mutex4.cpp
index 539a8be803c60583ce714ecf94b460c9783f62d7..f7dfab02353124381f100b0fa7cfe9e7bfe77830 100644
--- a/compiler-rt/test/tsan/custom_mutex4.cpp
+++ b/compiler-rt/test/tsan/custom_mutex4.cpp
@@ -1,7 +1,7 @@
-// RUN: %clangxx_tsan -O1 --std=c++11 %s -o %t && %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_tsan -O1 --std=c++17 %s -o %t && %run %t 2>&1 | FileCheck %s
 #include "custom_mutex.h"
 
-#include <type_traits>
+#include <cstddef>
 
 // Test that the destruction events of a mutex are ignored when the
 // annotations request this.
@@ -12,14 +12,14 @@
 // has run.
 
 int main() {
-  std::aligned_storage<sizeof(Mutex), alignof(Mutex)>::type mu1_store;
+  alignas(Mutex) std::byte mu1_store[sizeof(Mutex)];
   Mutex* mu1 = reinterpret_cast<Mutex*>(&mu1_store);
   new(&mu1_store) Mutex(false, __tsan_mutex_linker_init);
   mu1->Lock();
   mu1->~Mutex();
   mu1->Unlock();
 
-  std::aligned_storage<sizeof(Mutex), alignof(Mutex)>::type mu2_store;
+  alignas(Mutex) std::byte mu2_store[sizeof(Mutex)];
   Mutex* mu2 = reinterpret_cast<Mutex*>(&mu2_store);
   new(&mu2_store) Mutex(false, 0, __tsan_mutex_not_static);
   mu2->Lock();
diff --git a/compiler-rt/test/tsan/custom_mutex5.cpp b/compiler-rt/test/tsan/custom_mutex5.cpp
index cb18b235e26288cace8ac7e27a7045b996d61145..6d65829e5df3cc63a08f0fecd92b0e5059f5a23f 100644
--- a/compiler-rt/test/tsan/custom_mutex5.cpp
+++ b/compiler-rt/test/tsan/custom_mutex5.cpp
@@ -1,20 +1,20 @@
-// RUN: %clangxx_tsan -O1 --std=c++11 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_tsan -O1 --std=c++17 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s
 #include "custom_mutex.h"
 
-#include <type_traits>
+#include <cstddef>
 
 // Test that we detect the destruction of an in-use mutex when the
 // thread annotations don't otherwise disable the check.
 
 int main() {
-  std::aligned_storage<sizeof(Mutex), alignof(Mutex)>::type mu1_store;
+  alignas(Mutex) std::byte mu1_store[sizeof(Mutex)];
   Mutex* mu1 = reinterpret_cast<Mutex*>(&mu1_store);
   new(&mu1_store) Mutex(false, 0);
   mu1->Lock();
   mu1->~Mutex();
   mu1->Unlock();
 
-  std::aligned_storage<sizeof(Mutex), alignof(Mutex)>::type mu2_store;
+  alignas(Mutex) std::byte mu2_store[sizeof(Mutex)];
   Mutex* mu2 = reinterpret_cast<Mutex*>(&mu2_store);
   new(&mu2_store)
       Mutex(false, __tsan_mutex_not_static, __tsan_mutex_not_static);
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
index 0e20cfbbd264b232a08d54db414aad8546c7149f..17587b3f3e18d685ce593a2992290b6a6e6ebad8 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
@@ -10,6 +10,7 @@ import abc
 import imp
 import os
 import sys
+from enum import IntEnum
 from pathlib import PurePath, Path
 from collections import defaultdict, namedtuple
 
@@ -37,6 +38,26 @@ def _load_com_module():
 VSBreakpoint = namedtuple("VSBreakpoint", "path, line, col, cond")
 
 
+# Visual Studio events.
+# https://learn.microsoft.com/en-us/dotnet/api/envdte.dbgeventreason?view=visualstudiosdk-2022
+class DbgEvent(IntEnum):
+    dbgEventReasonNone = 1
+    dbgEventReasonGo = 2
+    dbgEventReasonAttachProgram = 3
+    dbgEventReasonDetachProgram = 4
+    dbgEventReasonLaunchProgram = 5
+    dbgEventReasonEndProgram = 6
+    dbgEventReasonStopDebugging = 7
+    dbgEventReasonStep = 8
+    dbgEventReasonBreakpoint = 9
+    dbgEventReasonExceptionThrown = 10
+    dbgEventReasonExceptionNotHandled = 11
+    dbgEventReasonUserBreak = 12
+    dbgEventReasonContextSwitch = 13
+
+    first = dbgEventReasonNone
+    last = dbgEventReasonContextSwitch
+
 class VisualStudio(
     DebuggerBase, metaclass=abc.ABCMeta
 ):  # pylint: disable=abstract-method
@@ -307,6 +328,20 @@ class VisualStudio(
                 )
             )
 
+    def _translate_stop_reason(self, reason):
+        if reason == DbgEvent.dbgEventReasonNone:
+            return None
+        if reason == DbgEvent.dbgEventReasonBreakpoint:
+            return StopReason.BREAKPOINT
+        if reason == DbgEvent.dbgEventReasonStep:
+            return StopReason.STEP
+        if reason == DbgEvent.dbgEventReasonEndProgram:
+            return StopReason.PROGRAM_EXIT
+        if reason == DbgEvent.dbgEventReasonExceptionNotHandled:
+            return StopReason.ERROR
+        assert reason <= DbgEvent.last and reason >= DbgEvent.first
+        return StopReason.OTHER
+
     def _get_step_info(self, watches, step_index):
         thread = self._debugger.CurrentThread
         stackframes = thread.StackFrames
@@ -347,16 +382,13 @@ class VisualStudio(
             frames[0].loc = loc
             state_frames[0].location = SourceLocation(**self._location)
 
-        reason = StopReason.BREAKPOINT
-        if loc.path is None:  # pylint: disable=no-member
-            reason = StopReason.STEP
-
+        stop_reason = self._translate_stop_reason(self._debugger.LastBreakReason)
         program_state = ProgramState(frames=state_frames)
 
         return StepIR(
             step_index=step_index,
             frames=frames,
-            stop_reason=reason,
+            stop_reason=stop_reason,
             program_state=program_state,
         )
 
diff --git a/flang/cmake/modules/AddFlangOffloadRuntime.cmake b/flang/cmake/modules/AddFlangOffloadRuntime.cmake
index 0af12c8cfd54ff116c11220c1677024b100bbc67..6407be5d038b8134a55120ba4448f6be85248056 100644
--- a/flang/cmake/modules/AddFlangOffloadRuntime.cmake
+++ b/flang/cmake/modules/AddFlangOffloadRuntime.cmake
@@ -101,6 +101,7 @@ macro(enable_omp_offload_compilation files)
         "gfx908;gfx90a;gfx90c;gfx940;gfx1010;gfx1030"
         "gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036"
         "gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151"
+        "gfx1152"
         )
       set(all_nvptx_architectures
         "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62"
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 41129b10083b17be7f7089615e7b246e055c2dca..8853d4d9e1c79cb7d25477ebcca803ac87dc0c6f 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -967,4 +967,35 @@ program test_etime
     print *, tarray(1)
     print *, tarray(2)
 end program test_etime
+```
+
+### Non-Standard Intrinsics: GETCWD
+
+#### Description
+`GETCWD(C, STATUS)` returns current working directory.
+
+This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit.
+
+*C* and *STATUS* are `INTENT(OUT)` and provide the following:
+
+|            |                                                                                                   |
+|------------|---------------------------------------------------------------------------------------------------|
+| `C`        | Current work directory. The type shall be `CHARACTER` and of default kind.       |
+| `STATUS`   | (Optional) Status flag. Returns 0 on success, a system specific and nonzero error code otherwise. The type shall be `INTEGER` and of a kind greater or equal to 4. |
+
+#### Usage and Info
+
+- **Standard:** GNU extension
+- **Class:** Subroutine, function
+- **Syntax:** `CALL GETCWD(C, STATUS)`, `STATUS = GETCWD(C)`
+
+#### Example
+```Fortran
+PROGRAM example_getcwd
+  CHARACTER(len=255) :: cwd
+  INTEGER :: status
+  CALL getcwd(cwd, status)
+  PRINT *, cwd
+  PRINT *, status
+END PROGRAM
 ```
\ No newline at end of file
diff --git a/flang/include/flang/Optimizer/Builder/BoxValue.h b/flang/include/flang/Optimizer/Builder/BoxValue.h
index 5c7e89dbc08f110d921d2e80466b97974869e465..9fdaa0b19717963d49507359b103b9cb93f03ef0 100644
--- a/flang/include/flang/Optimizer/Builder/BoxValue.h
+++ b/flang/include/flang/Optimizer/Builder/BoxValue.h
@@ -433,7 +433,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &, const ExtendedValue &);
 /// substituted.
 ExtendedValue substBase(const ExtendedValue &exv, mlir::Value base);
 
-/// Is the extended value `exv` an array?
+/// Is the extended value `exv` an array? Note that this returns true for
+/// assumed-ranks that could actually be scalars at runtime.
 bool isArray(const ExtendedValue &exv);
 
 /// Get the type parameters for `exv`.
@@ -527,6 +528,15 @@ public:
                  [](const auto &box) -> bool { return false; });
   }
 
+  bool hasAssumedRank() const {
+    return match(
+        [](const fir::BoxValue &box) -> bool { return box.hasAssumedRank(); },
+        [](const fir::MutableBoxValue &box) -> bool {
+          return box.hasAssumedRank();
+        },
+        [](const auto &box) -> bool { return false; });
+  }
+
   /// LLVM style debugging of extended values
   LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
 
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index c47e41eab18b2a3949e66423a0d886eb76aba40a..8ef5d59b92f0c1b6aa2da50f7f108c5643d6fe7f 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -232,6 +232,8 @@ struct IntrinsicLibrary {
   mlir::Value genFloor(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genFraction(mlir::Type resultType,
                           mlir::ArrayRef<mlir::Value> args);
+  fir::ExtendedValue genGetCwd(std::optional<mlir::Type> resultType,
+                               llvm::ArrayRef<fir::ExtendedValue> args);
   void genGetCommand(mlir::ArrayRef<fir::ExtendedValue> args);
   mlir::Value genGetPID(mlir::Type resultType,
                         llvm::ArrayRef<mlir::Value> args);
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Command.h b/flang/include/flang/Optimizer/Builder/Runtime/Command.h
index 976fb3aa0b6fbb729464a12fcd9edb8c0cd8e8ad..0d60a367d99981308dd00b3b9b90c61fc3cfbbad 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Command.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Command.h
@@ -53,5 +53,10 @@ mlir::Value genGetEnvVariable(fir::FirOpBuilder &, mlir::Location,
                               mlir::Value length, mlir::Value trimName,
                               mlir::Value errmsg);
 
+/// Generate a call to the GetCwd runtime function which implements
+/// the GETCWD intrinsic.
+mlir::Value genGetCwd(fir::FirOpBuilder &builder, mlir::Location loc,
+                      mlir::Value c);
+
 } // namespace fir::runtime
 #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_COMMAND_H
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Inquiry.h b/flang/include/flang/Optimizer/Builder/Runtime/Inquiry.h
index 132592a0197f8fd0b14143e215c5cb43f3245e85..5f14d7781004b3c6f25b9a0ac2868640c56c227e 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Inquiry.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Inquiry.h
@@ -32,6 +32,12 @@ mlir::Value genLboundDim(fir::FirOpBuilder &builder, mlir::Location loc,
 void genUbound(fir::FirOpBuilder &builder, mlir::Location loc,
                mlir::Value resultBox, mlir::Value array, mlir::Value kind);
 
+/// Generate call to `Shape` runtime routine.
+/// First argument is a raw pointer to the result array storage that
+/// must be allocated by the caller.
+void genShape(fir::FirOpBuilder &builder, mlir::Location loc,
+              mlir::Value resultAddr, mlir::Value arrayt, mlir::Value kind);
+
 /// Generate call to `Size` runtime routine. This routine is a specialized
 /// version when the DIM argument is not specified by the user.
 mlir::Value genSize(fir::FirOpBuilder &builder, mlir::Location loc,
diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index 0c34b640a5c9c5c159ee5ed5eaebcb1673df80ac..aedb6769186e9683b7e80b48594e6213d10a0d27 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -67,6 +67,36 @@ def fir_BoxFieldAttr : I32EnumAttr<
   let cppNamespace = "fir";
 }
 
+def fir_ReduceOperationEnum : I32BitEnumAttr<"ReduceOperationEnum",
+    "intrinsic operations and functions supported by DO CONCURRENT REDUCE",
+    [
+      I32BitEnumAttrCaseBit<"Add", 0, "add">,
+      I32BitEnumAttrCaseBit<"Multiply", 1, "multiply">,
+      I32BitEnumAttrCaseBit<"AND", 2, "and">,
+      I32BitEnumAttrCaseBit<"OR", 3, "or">,
+      I32BitEnumAttrCaseBit<"EQV", 4, "eqv">,
+      I32BitEnumAttrCaseBit<"NEQV", 5, "neqv">,
+      I32BitEnumAttrCaseBit<"MAX", 6, "max">,
+      I32BitEnumAttrCaseBit<"MIN", 7, "min">,
+      I32BitEnumAttrCaseBit<"IAND", 8, "iand">,
+      I32BitEnumAttrCaseBit<"IOR", 9, "ior">,
+      I32BitEnumAttrCaseBit<"EIOR", 10, "eior">
+    ]> {
+  let separator = ", ";
+  let cppNamespace = "::fir";
+  let printBitEnumPrimaryGroups = 1;
+}
+
+def fir_ReduceAttr : fir_Attr<"Reduce"> {
+  let mnemonic = "reduce_attr";
+
+  let parameters = (ins
+    "ReduceOperationEnum":$reduce_operation
+  );
+
+  let assemblyFormat = "`<` $reduce_operation `>`";
+}
+
 // mlir::SideEffects::Resource for modelling operations which add debugging information
 def DebuggingResource : Resource<"::fir::DebuggingResource">;
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 37fbd1f9692a467ac182b4415389ccb18fb6305c..e7da3af5485ccaf36668406ccaeb000a1857eeb4 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2125,8 +2125,8 @@ class region_Op<string mnemonic, list<Trait> traits = []> :
   let hasVerifier = 1;
 }
 
-def fir_DoLoopOp : region_Op<"do_loop",
-    [DeclareOpInterfaceMethods<LoopLikeOpInterface,
+def fir_DoLoopOp : region_Op<"do_loop", [AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<LoopLikeOpInterface,
         ["getYieldedValuesMutable"]>]> {
   let summary = "generalized loop operation";
   let description = [{
@@ -2156,9 +2156,11 @@ def fir_DoLoopOp : region_Op<"do_loop",
     Index:$lowerBound,
     Index:$upperBound,
     Index:$step,
+    Variadic<AnyType>:$reduceOperands,
     Variadic<AnyType>:$initArgs,
     OptionalAttr<UnitAttr>:$unordered,
-    OptionalAttr<UnitAttr>:$finalValue
+    OptionalAttr<UnitAttr>:$finalValue,
+    OptionalAttr<ArrayAttr>:$reduceAttrs
   );
   let results = (outs Variadic<AnyType>:$results);
   let regions = (region SizedRegion<1>:$region);
@@ -2169,6 +2171,8 @@ def fir_DoLoopOp : region_Op<"do_loop",
       "mlir::Value":$step, CArg<"bool", "false">:$unordered,
       CArg<"bool", "false">:$finalCountValue,
       CArg<"mlir::ValueRange", "std::nullopt">:$iterArgs,
+      CArg<"mlir::ValueRange", "std::nullopt">:$reduceOperands,
+      CArg<"llvm::ArrayRef<mlir::Attribute>", "{}">:$reduceAttrs,
       CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>
   ];
 
@@ -2181,11 +2185,12 @@ def fir_DoLoopOp : region_Op<"do_loop",
       return getBody()->getArguments().drop_front();
     }
     mlir::Operation::operand_range getIterOperands() {
-      return getOperands().drop_front(getNumControlOperands());
+      return getOperands()
+          .drop_front(getNumControlOperands() + getNumReduceOperands());
     }
     llvm::MutableArrayRef<mlir::OpOperand> getInitsMutable() {
-      return
-          getOperation()->getOpOperands().drop_front(getNumControlOperands());
+      return getOperation()->getOpOperands()
+          .drop_front(getNumControlOperands() + getNumReduceOperands());
     }
 
     void setLowerBound(mlir::Value bound) { (*this)->setOperand(0, bound); }
@@ -2200,11 +2205,25 @@ def fir_DoLoopOp : region_Op<"do_loop",
     unsigned getNumControlOperands() { return 3; }
     /// Does the operation hold operands for loop-carried values
     bool hasIterOperands() {
-      return (*this)->getNumOperands() > getNumControlOperands();
+      return getNumIterOperands() > 0;
+    }
+    /// Does the operation hold operands for reduction variables
+    bool hasReduceOperands() {
+      return getNumReduceOperands() > 0;
+    }
+    /// Get Number of variadic operands
+    unsigned getNumOperands(unsigned idx) {
+      auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
+        getOperandSegmentSizeAttr());
+      return static_cast<unsigned>(segments[idx]);
+    }
+    // Get Number of reduction operands
+    unsigned getNumReduceOperands() {
+      return getNumOperands(3);
     }
     /// Get Number of loop-carried values
     unsigned getNumIterOperands() {
-      return (*this)->getNumOperands() - getNumControlOperands();
+      return getNumOperands(4);
     }
 
     /// Get the body of the loop
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index a7ba704fdb39b84d1845635b71d45a41bc5a7f1c..9fa819e2bf502e2ed89eb85ddcb6a2a0c7a2f2fc 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -54,15 +54,14 @@ namespace fir {
 #define GEN_PASS_DECL_OMPMAPINFOFINALIZATIONPASS
 #define GEN_PASS_DECL_OMPMARKDECLARETARGETPASS
 #define GEN_PASS_DECL_OMPFUNCTIONFILTERING
+#define GEN_PASS_DECL_VSCALEATTR
+#define GEN_PASS_DECL_FUNCTIONATTR
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
 std::unique_ptr<mlir::Pass> createCFGConversionPassWithNSW();
-std::unique_ptr<mlir::Pass> createExternalNameConversionPass();
-std::unique_ptr<mlir::Pass>
-createExternalNameConversionPass(bool appendUnderscore);
 std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
 std::unique_ptr<mlir::Pass>
@@ -77,17 +76,6 @@ std::unique_ptr<mlir::Pass> createVScaleAttrPass();
 std::unique_ptr<mlir::Pass>
 createVScaleAttrPass(std::pair<unsigned, unsigned> vscaleAttr);
 
-struct FunctionAttrTypes {
-  mlir::LLVM::framePointerKind::FramePointerKind framePointerKind =
-      mlir::LLVM::framePointerKind::FramePointerKind::None;
-};
-
-std::unique_ptr<mlir::Pass> createFunctionAttrPass();
-std::unique_ptr<mlir::Pass>
-createFunctionAttrPass(FunctionAttrTypes &functionAttr, bool noInfsFPMath,
-                       bool noNaNsFPMath, bool approxFuncFPMath,
-                       bool noSignedZerosFPMath, bool unsafeFPMath);
-
 void populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
                                    bool forceLoopToExecuteOnce = false,
                                    bool setNSW = false);
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 82638200e5e207594ed10fcee11fbaeab97ca1da..7a3baca4c19dac97182c13666082841b269c5033 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -163,7 +163,6 @@ def ExternalNameConversion : Pass<"external-name-interop", "mlir::ModuleOp"> {
   let description = [{
     Demangle FIR internal name and mangle them for external interoperability.
   }];
-  let constructor = "::fir::createExternalNameConversionPass()";
   let options = [
     Option<"appendUnderscoreOpt", "append-underscore",
            "bool", /*default=*/"true",
@@ -360,7 +359,6 @@ def VScaleAttr : Pass<"vscale-attr", "mlir::func::FuncOp"> {
            "std::pair<unsigned, unsigned>", /*default=*/"std::pair<unsigned, unsigned>{}",
            "vector scale range">,
   ];
-  let constructor = "::fir::createVScaleAttrPass()";
 }
 
 def FunctionAttr : Pass<"function-attr", "mlir::func::FuncOp"> {
@@ -396,7 +394,6 @@ def FunctionAttr : Pass<"function-attr", "mlir::func::FuncOp"> {
            "bool", /*default=*/"false",
            "Set the unsafe-fp-math attribute on functions in the module.">,
   ];
-  let constructor = "::fir::createFunctionAttrPass()";
 }
 
 def AssumedRankOpConversion : Pass<"fir-assumed-rank-op", "mlir::ModuleOp"> {
diff --git a/flang/include/flang/Runtime/command.h b/flang/include/flang/Runtime/command.h
index c67d171c8e2f1b89a605428e74b01a305ab7fb8b..7ab3f6442dcf92ec9074c124e860ba6993345cb3 100644
--- a/flang/include/flang/Runtime/command.h
+++ b/flang/include/flang/Runtime/command.h
@@ -55,6 +55,10 @@ std::int32_t RTNAME(GetEnvVariable)(const Descriptor &name,
     const Descriptor *value = nullptr, const Descriptor *length = nullptr,
     bool trim_name = true, const Descriptor *errmsg = nullptr,
     const char *sourceFile = nullptr, int line = 0);
+
+// Calls getcwd()
+std::int32_t RTNAME(GetCwd)(
+    const Descriptor &cwd, const char *sourceFile, int line);
 }
 } // namespace Fortran::runtime
 
diff --git a/flang/include/flang/Runtime/inquiry.h b/flang/include/flang/Runtime/inquiry.h
index 3fe670b0fae31ebad214e810a564a03df51c5cf6..dde6e722ad6e955787096c30523db5cca8aa0fcd 100644
--- a/flang/include/flang/Runtime/inquiry.h
+++ b/flang/include/flang/Runtime/inquiry.h
@@ -23,13 +23,21 @@ extern "C" {
 
 std::int64_t RTDECL(LboundDim)(const Descriptor &array, int dim,
     const char *sourceFile = nullptr, int line = 0);
-void RTDECL(Ubound)(Descriptor &result, const Descriptor &array, int kind,
+
+void RTDECL(Lbound)(void *result, const Descriptor &array, int kind,
+    const char *sourceFile = nullptr, int line = 0);
+
+void RTDECL(Shape)(void *result, const Descriptor &array, int kind,
     const char *sourceFile = nullptr, int line = 0);
 std::int64_t RTDECL(Size)(
     const Descriptor &array, const char *sourceFile = nullptr, int line = 0);
+
 std::int64_t RTDECL(SizeDim)(const Descriptor &array, int dim,
     const char *sourceFile = nullptr, int line = 0);
 
+void RTDECL(Ubound)(Descriptor &result, const Descriptor &array, int kind,
+    const char *sourceFile = nullptr, int line = 0);
+
 } // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_INQUIRY_H_
diff --git a/flang/include/flang/Runtime/magic-numbers.h b/flang/include/flang/Runtime/magic-numbers.h
index 38ccc5e7d3df659fba20016b3523f50d491dd53c..1cded1fd63238f3b35efda5669e9307d5fd37b8b 100644
--- a/flang/include/flang/Runtime/magic-numbers.h
+++ b/flang/include/flang/Runtime/magic-numbers.h
@@ -68,6 +68,11 @@ Additional status code for a bad pointer DEALLOCATE.
 #endif
 #define FORTRAN_RUNTIME_STAT_BAD_POINTER_DEALLOCATION 110
 
+#if 0
+Status codes for GETCWD.
+#endif
+#define FORTRAN_RUNTIME_STAT_MISSING_CWD 111
+
 #if 0
 ieee_class_type values
 The sequence is that of F18 Clause 17.2p3, but nothing depends on that.
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index c5c35e9a6a33f2cf3d33b90855a7899b9b573e05..2a0cfc04aa3509944879a280b01785bea71610fc 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -233,9 +233,8 @@ inline void addBoxedProcedurePass(mlir::PassManager &pm) {
 
 inline void addExternalNameConversionPass(
     mlir::PassManager &pm, bool appendUnderscore = true) {
-  addPassConditionally(pm, disableExternalNameConversion, [&]() {
-    return fir::createExternalNameConversionPass(appendUnderscore);
-  });
+  addPassConditionally(pm, disableExternalNameConversion,
+      [&]() { return fir::createExternalNameConversion({appendUnderscore}); });
 }
 
 // Use inliner extension point callback to register the default inliner pass.
@@ -370,27 +369,25 @@ inline void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
   fir::createDebugPasses(pm, config.DebugInfo, config.OptLevel, inputFilename);
 
   if (config.VScaleMin != 0)
-    pm.addPass(fir::createVScaleAttrPass({config.VScaleMin, config.VScaleMax}));
+    pm.addPass(fir::createVScaleAttr({{config.VScaleMin, config.VScaleMax}}));
 
   // Add function attributes
-  fir::FunctionAttrTypes functionAttrs;
+  mlir::LLVM::framePointerKind::FramePointerKind framePointerKind;
 
   if (config.FramePointerKind != llvm::FramePointerKind::None ||
       config.NoInfsFPMath || config.NoNaNsFPMath || config.ApproxFuncFPMath ||
       config.NoSignedZerosFPMath || config.UnsafeFPMath) {
     if (config.FramePointerKind == llvm::FramePointerKind::NonLeaf)
-      functionAttrs.framePointerKind =
+      framePointerKind =
           mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
     else if (config.FramePointerKind == llvm::FramePointerKind::All)
-      functionAttrs.framePointerKind =
-          mlir::LLVM::framePointerKind::FramePointerKind::All;
+      framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
     else
-      functionAttrs.framePointerKind =
-          mlir::LLVM::framePointerKind::FramePointerKind::None;
+      framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
 
-    pm.addPass(fir::createFunctionAttrPass(functionAttrs, config.NoInfsFPMath,
+    pm.addPass(fir::createFunctionAttr({framePointerKind, config.NoInfsFPMath,
         config.NoNaNsFPMath, config.ApproxFuncFPMath,
-        config.NoSignedZerosFPMath, config.UnsafeFPMath));
+        config.NoSignedZerosFPMath, config.UnsafeFPMath}));
   }
 
   fir::addFIRToLLVMPass(pm, config);
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 12d13422574b569f2f42fe0d232e68318c0a1b0b..58c1b6989f495afadbee449165e248f8b3f7ce1c 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -514,6 +514,10 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"gamma", {{"x", SameReal}}, SameReal},
     {"get_team", {{"level", DefaultInt, Rank::scalar, Optionality::optional}},
         TeamType, Rank::scalar, IntrinsicClass::transformationalFunction},
+    {"getcwd",
+        {{"c", DefaultChar, Rank::scalar, Optionality::required,
+            common::Intent::Out}},
+        TypePattern{IntType, KindCode::greaterOrEqualToKind, 4}},
     {"getpid", {}, DefaultInt},
     {"huge",
         {{"x", SameIntOrReal, Rank::anyOrAssumedRank, Optionality::required,
@@ -1406,6 +1410,12 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"errmsg", DefaultChar, Rank::scalar, Optionality::optional,
                 common::Intent::InOut}},
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
+    {"getcwd",
+        {{"c", DefaultChar, Rank::scalar, Optionality::required,
+             common::Intent::Out},
+            {"status", TypePattern{IntType, KindCode::greaterOrEqualToKind, 4},
+                Rank::scalar, Optionality::optional, common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"move_alloc",
         {{"from", SameType, Rank::known, Optionality::required,
              common::Intent::InOut},
@@ -2574,7 +2584,7 @@ bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
     const std::string &name) const {
   // Collection for some intrinsics with function and subroutine form,
   // in order to pass the semantic check.
-  static const std::string dualIntrinsic[]{{"etime"}};
+  static const std::string dualIntrinsic[]{{"etime"}, {"getcwd"}};
 
   return std::find_if(std::begin(dualIntrinsic), std::end(dualIntrinsic),
              [&name](const std::string &dualName) {
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index f7cd0ea83ad12ea5c0ee00ccf508aa17979c3515..98fb5dcf7722e28d93875f5158937695a530a914 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -36,30 +36,64 @@ struct TypeTy : public evaluate::SomeType {
   bool operator==(const TypeTy &t) const { return true; }
 };
 
-using IdTy = semantics::Symbol *;
+template <typename ExprTy>
+struct IdTyTemplate {
+  // "symbol" is always non-null for id's of actual objects.
+  Fortran::semantics::Symbol *symbol;
+  std::optional<ExprTy> designator;
+
+  bool operator==(const IdTyTemplate &other) const {
+    // If symbols are different, then the objects are different.
+    if (symbol != other.symbol)
+      return false;
+    if (symbol == nullptr)
+      return true;
+    // Equal symbols don't necessarily indicate identical objects,
+    // for example, a derived object component may use a single symbol,
+    // which will refer to different objects for different designators,
+    // e.g. a%c and b%c.
+    return designator == other.designator;
+  }
+
+  operator bool() const { return symbol != nullptr; }
+};
+
 using ExprTy = SomeExpr;
 
 template <typename T>
 using List = tomp::ListT<T>;
 } // namespace Fortran::lower::omp
 
+// Specialization of the ObjectT template
 namespace tomp::type {
 template <>
-struct ObjectT<Fortran::lower::omp::IdTy, Fortran::lower::omp::ExprTy> {
-  using IdTy = Fortran::lower::omp::IdTy;
+struct ObjectT<Fortran::lower::omp::IdTyTemplate<Fortran::lower::omp::ExprTy>,
+               Fortran::lower::omp::ExprTy> {
+  using IdTy = Fortran::lower::omp::IdTyTemplate<Fortran::lower::omp::ExprTy>;
   using ExprTy = Fortran::lower::omp::ExprTy;
 
-  IdTy id() const { return symbol; }
-  Fortran::semantics::Symbol *sym() const { return symbol; }
-  const std::optional<ExprTy> &ref() const { return designator; }
+  IdTy id() const { return identity; }
+  Fortran::semantics::Symbol *sym() const { return identity.symbol; }
+  const std::optional<ExprTy> &ref() const { return identity.designator; }
 
-  IdTy symbol;
-  std::optional<ExprTy> designator;
+  IdTy identity;
 };
 } // namespace tomp::type
 
 namespace Fortran::lower::omp {
+using IdTy = IdTyTemplate<ExprTy>;
+}
 
+namespace std {
+template <>
+struct hash<Fortran::lower::omp::IdTy> {
+  size_t operator()(const Fortran::lower::omp::IdTy &id) const {
+    return static_cast<size_t>(reinterpret_cast<uintptr_t>(id.symbol));
+  }
+};
+} // namespace std
+
+namespace Fortran::lower::omp {
 using Object = tomp::ObjectT<IdTy, ExprTy>;
 using ObjectList = tomp::ObjectListT<IdTy, ExprTy>;
 
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index b206040c237c586f7f59c1f87f4193a2ff71fd82..e2b55fcc64062578fee74b15988108bd5b42028e 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -48,14 +48,13 @@ DataSharingProcessor::DataSharingProcessor(
 }
 
 void DataSharingProcessor::processStep1(
-    mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+    mlir::omp::PrivateClauseOps *clauseOps) {
   collectSymbolsForPrivatization();
   collectDefaultSymbols();
   collectImplicitSymbols();
   collectPreDeterminedSymbols();
 
-  privatize(clauseOps, privateSyms);
+  privatize(clauseOps);
 
   insertBarrier();
 }
@@ -415,16 +414,14 @@ void DataSharingProcessor::collectPreDeterminedSymbols() {
                    preDeterminedSymbols);
 }
 
-void DataSharingProcessor::privatize(
-    mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) {
   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
     if (const auto *commonDet =
             sym->detailsIf<semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects())
-        doPrivatize(&*mem, clauseOps, privateSyms);
+        doPrivatize(&*mem, clauseOps);
     } else
-      doPrivatize(sym, clauseOps, privateSyms);
+      doPrivatize(sym, clauseOps);
   }
 }
 
@@ -441,9 +438,8 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
     }
 }
 
-void DataSharingProcessor::doPrivatize(
-    const semantics::Symbol *sym, mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
+                                       mlir::omp::PrivateClauseOps *clauseOps) {
   if (!useDelayedPrivatization) {
     cloneSymbol(sym);
     copyFirstPrivateSymbol(sym);
@@ -548,9 +544,6 @@ void DataSharingProcessor::doPrivatize(
     clauseOps->privateVars.push_back(hsb.getAddr());
   }
 
-  if (privateSyms)
-    privateSyms->push_back(sym);
-
   symToPrivatizer[sym] = privatizerOp;
 }
 
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 80a956de35ba9dd30439ba5bed810b4ea05741b8..7c0c831ed43a827e3274464045d0d26a7d1a30df 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -44,7 +44,9 @@ private:
     void Post(const T &) {}
 
     bool Pre(const parser::OpenMPConstruct &omp) {
-      currentConstruct = &omp;
+      // Skip constructs that may not have privatizations.
+      if (!std::holds_alternative<parser::OpenMPCriticalConstruct>(omp.u))
+        currentConstruct = &omp;
       return true;
     }
 
@@ -103,18 +105,15 @@ private:
   void collectDefaultSymbols();
   void collectImplicitSymbols();
   void collectPreDeterminedSymbols();
-  void privatize(mlir::omp::PrivateClauseOps *clauseOps,
-                 llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
+  void privatize(mlir::omp::PrivateClauseOps *clauseOps);
   void defaultPrivatize(
       mlir::omp::PrivateClauseOps *clauseOps,
       llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
   void implicitPrivatize(
       mlir::omp::PrivateClauseOps *clauseOps,
       llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
-  void
-  doPrivatize(const semantics::Symbol *sym,
-              mlir::omp::PrivateClauseOps *clauseOps,
-              llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
+  void doPrivatize(const semantics::Symbol *sym,
+                   mlir::omp::PrivateClauseOps *clauseOps);
   void copyLastPrivatize(mlir::Operation *op);
   void insertLastPrivateCompare(mlir::Operation *op);
   void cloneSymbol(const semantics::Symbol *sym);
@@ -145,15 +144,18 @@ public:
   // Step2 performs the copying for lastprivates and requires knowledge of the
   // MLIR operation to insert the last private update. Step2 adds
   // dealocation code as well.
-  void processStep1(
-      mlir::omp::PrivateClauseOps *clauseOps = nullptr,
-      llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms = nullptr);
+  void processStep1(mlir::omp::PrivateClauseOps *clauseOps = nullptr);
   void processStep2(mlir::Operation *op, bool isLoop);
 
   void setLoopIV(mlir::Value iv) {
     assert(!loopIV && "Loop iteration variable already set");
     loopIV = iv;
   }
+
+  const llvm::SetVector<const semantics::Symbol *> &
+  getAllSymbolsToPrivatize() const {
+    return allPrivatizedSymbols;
+  }
 };
 
 } // namespace omp
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f84440d95ec116465f7c2b11b6b4e8ebe710f9d4..6b391e11beb481d31694fa126c30640aee42c41a 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -456,6 +456,33 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
+/// For an operation that takes `omp.private` values as region args, this util
+/// merges the private vars info into the region arguments list.
+///
+/// \tparam OMPOP  - the OpenMP op that takes `omp.private` inputs.
+/// \tparam InfoTy - the type of private info we want to merge; e.g. mlir::Type
+/// or mlir::Location fields of the private var list.
+///
+/// \param [in] op                 - the op accepting `omp.private` inputs.
+/// \param [in] currentList        - the current list of region info that we
+/// want to merge private info with. For example this could be the list of types
+/// or locations of previous arguments to \op's region.
+/// \param [in] infoAccessor       - for a private variable, this returns the
+/// data we want to merge: type or location.
+/// \param [out] allRegionArgsInfo - the merged list of region info.
+template <typename OMPOp, typename InfoTy>
+static void
+mergePrivateVarsInfo(OMPOp op, llvm::ArrayRef<InfoTy> currentList,
+                     llvm::function_ref<InfoTy(mlir::Value)> infoAccessor,
+                     llvm::SmallVectorImpl<InfoTy> &allRegionArgsInfo) {
+  mlir::OperandRange privateVars = op.getPrivateVars();
+
+  llvm::transform(currentList, std::back_inserter(allRegionArgsInfo),
+                  [](InfoTy i) { return i; });
+  llvm::transform(privateVars, std::back_inserter(allRegionArgsInfo),
+                  infoAccessor);
+}
+
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -758,6 +785,7 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                   llvm::ArrayRef<const semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
                   llvm::ArrayRef<mlir::Type> mapSymTypes,
+                  DataSharingProcessor &dsp,
                   const mlir::Location &currentLocation,
                   const ConstructQueue &queue, ConstructQueue::iterator item) {
   assert(mapSymTypes.size() == mapSymLocs.size());
@@ -765,8 +793,20 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Region &region = targetOp.getRegion();
 
-  auto *regionBlock =
-      firOpBuilder.createBlock(&region, {}, mapSymTypes, mapSymLocs);
+  llvm::SmallVector<mlir::Type> allRegionArgTypes;
+  mergePrivateVarsInfo(targetOp, mapSymTypes,
+                       llvm::function_ref<mlir::Type(mlir::Value)>{
+                           [](mlir::Value v) { return v.getType(); }},
+                       allRegionArgTypes);
+
+  llvm::SmallVector<mlir::Location> allRegionArgLocs;
+  mergePrivateVarsInfo(targetOp, mapSymLocs,
+                       llvm::function_ref<mlir::Location(mlir::Value)>{
+                           [](mlir::Value v) { return v.getLoc(); }},
+                       allRegionArgLocs);
+
+  auto *regionBlock = firOpBuilder.createBlock(&region, {}, allRegionArgTypes,
+                                               allRegionArgLocs);
 
   // Clones the `bounds` placing them inside the target region and returns them.
   auto cloneBound = [&](mlir::Value bound) {
@@ -830,6 +870,20 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
         });
   }
 
+  for (auto [argIndex, argSymbol] :
+       llvm::enumerate(dsp.getAllSymbolsToPrivatize())) {
+    argIndex = mapSyms.size() + argIndex;
+
+    const mlir::BlockArgument &arg = region.getArgument(argIndex);
+    converter.bindSymbol(*argSymbol,
+                         hlfir::translateToExtendedValue(
+                             currentLocation, firOpBuilder, hlfir::Entity{arg},
+                             /*contiguousHint=*/
+                             evaluate::IsSimplyContiguous(
+                                 *argSymbol, converter.getFoldingContext()))
+                             .first);
+  }
+
   // Check if cloning the bounds introduced any dependency on the outer region.
   // If so, then either clone them as well if they are MemoryEffectFree, or else
   // copy them to a new temporary and add them to the map and block_argument
@@ -907,6 +961,8 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   } else {
     genNestedEvaluations(converter, eval);
   }
+
+  dsp.processStep2(targetOp, /*isLoop=*/false);
 }
 
 template <typename OpTy, typename... Args>
@@ -1048,15 +1104,18 @@ static void genTargetClauses(
                         devicePtrSyms);
   cp.processMap(loc, stmtCtx, clauseOps, &mapSyms, &mapLocs, &mapTypes);
   cp.processThreadLimit(stmtCtx, clauseOps);
-  // TODO Support delayed privatization.
 
   if (processHostOnlyClauses)
     cp.processNowait(clauseOps);
 
   cp.processTODO<clause::Allocate, clause::Defaultmap, clause::Firstprivate,
-                 clause::InReduction, clause::Private, clause::Reduction,
+                 clause::InReduction, clause::Reduction,
                  clause::UsesAllocators>(loc,
                                          llvm::omp::Directive::OMPD_target);
+
+  // `target private(..)` is only supported in delayed privatization mode.
+  if (!enableDelayedPrivatizationStaging)
+    cp.processTODO<clause::Private>(loc, llvm::omp::Directive::OMPD_target);
 }
 
 static void genTargetDataClauses(
@@ -1289,7 +1348,6 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
-  llvm::SmallVector<const semantics::Symbol *> privateSyms;
   llvm::SmallVector<mlir::Type> reductionTypes;
   llvm::SmallVector<const semantics::Symbol *> reductionSyms;
   genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
@@ -1319,7 +1377,7 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                            /*useDelayedPrivatization=*/true, &symTable);
 
   if (privatize)
-    dsp.processStep1(&clauseOps, &privateSyms);
+    dsp.processStep1(&clauseOps);
 
   auto genRegionEntryCB = [&](mlir::Operation *op) {
     auto parallelOp = llvm::cast<mlir::omp::ParallelOp>(op);
@@ -1327,26 +1385,27 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
     llvm::SmallVector<mlir::Location> reductionLocs(
         clauseOps.reductionVars.size(), loc);
 
-    mlir::OperandRange privateVars = parallelOp.getPrivateVars();
-    mlir::Region &region = parallelOp.getRegion();
-
-    llvm::SmallVector<mlir::Type> privateVarTypes = reductionTypes;
-    privateVarTypes.reserve(privateVarTypes.size() + privateVars.size());
-    llvm::transform(privateVars, std::back_inserter(privateVarTypes),
-                    [](mlir::Value v) { return v.getType(); });
+    llvm::SmallVector<mlir::Type> allRegionArgTypes;
+    mergePrivateVarsInfo(parallelOp, llvm::ArrayRef(reductionTypes),
+                         llvm::function_ref<mlir::Type(mlir::Value)>{
+                             [](mlir::Value v) { return v.getType(); }},
+                         allRegionArgTypes);
 
-    llvm::SmallVector<mlir::Location> privateVarLocs = reductionLocs;
-    privateVarLocs.reserve(privateVarLocs.size() + privateVars.size());
-    llvm::transform(privateVars, std::back_inserter(privateVarLocs),
-                    [](mlir::Value v) { return v.getLoc(); });
+    llvm::SmallVector<mlir::Location> allRegionArgLocs;
+    mergePrivateVarsInfo(parallelOp, llvm::ArrayRef(reductionLocs),
+                         llvm::function_ref<mlir::Location(mlir::Value)>{
+                             [](mlir::Value v) { return v.getLoc(); }},
+                         allRegionArgLocs);
 
-    firOpBuilder.createBlock(&region, /*insertPt=*/{}, privateVarTypes,
-                             privateVarLocs);
+    mlir::Region &region = parallelOp.getRegion();
+    firOpBuilder.createBlock(&region, /*insertPt=*/{}, allRegionArgTypes,
+                             allRegionArgLocs);
 
     llvm::SmallVector<const semantics::Symbol *> allSymbols = reductionSyms;
-    allSymbols.append(privateSyms);
+    allSymbols.append(dsp.getAllSymbolsToPrivatize().begin(),
+                      dsp.getAllSymbolsToPrivatize().end());
+
     for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) {
-      fir::ExtendedValue hostExV = converter.getSymbolExtendedValue(*arg);
       converter.bindSymbol(*arg, hlfir::translateToExtendedValue(
                                      loc, firOpBuilder, hlfir::Entity{prv},
                                      /*contiguousHint=*/
@@ -1541,11 +1600,22 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    deviceAddrLocs, deviceAddrTypes, devicePtrSyms,
                    devicePtrLocs, devicePtrTypes);
 
+  llvm::SmallVector<const semantics::Symbol *> privateSyms;
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           /*shouldCollectPreDeterminedSymbols=*/
+                           lower::omp::isLastItemInQueue(item, queue),
+                           /*useDelayedPrivatization=*/true, &symTable);
+  dsp.processStep1(&clauseOps);
+
   // 5.8.1 Implicit Data-Mapping Attribute Rules
   // The following code follows the implicit data-mapping rules to map all the
-  // symbols used inside the region that have not been explicitly mapped using
-  // the map clause.
+  // symbols used inside the region that do not have explicit data-environment
+  // attribute clauses (neither data-sharing; e.g. `private`, nor `map`
+  // clauses).
   auto captureImplicitMap = [&](const semantics::Symbol &sym) {
+    if (dsp.getAllSymbolsToPrivatize().contains(&sym))
+      return;
+
     if (llvm::find(mapSyms, &sym) == mapSyms.end()) {
       mlir::Value baseOp = converter.getSymbolAddress(sym);
       if (!baseOp)
@@ -1632,7 +1702,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
   genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms,
-                    mapLocs, mapTypes, loc, queue, item);
+                    mapLocs, mapTypes, dsp, loc, queue, item);
   return targetOp;
 }
 
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index eff915f569f27ce15df95414eb276bf1e2cffaf0..36d96f37ff36a6cf9d88b31d0fbdf7ab91daf859 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -36,6 +36,12 @@ llvm::cl::opt<bool> enableDelayedPrivatization(
         "Emit `[first]private` variables as clauses on the MLIR ops."),
     llvm::cl::init(false));
 
+llvm::cl::opt<bool> enableDelayedPrivatizationStaging(
+    "openmp-enable-delayed-privatization-staging",
+    llvm::cl::desc("For partially supported constructs, emit `[first]private` "
+                   "variables as clauses on the MLIR ops."),
+    llvm::cl::init(false));
+
 namespace Fortran {
 namespace lower {
 namespace omp {
@@ -188,7 +194,7 @@ void addChildIndexAndMapToParent(
     std::map<const semantics::Symbol *,
              llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
     mlir::omp::MapInfoOp &mapOp, semantics::SemanticsContext &semaCtx) {
-  std::optional<evaluate::DataRef> dataRef = ExtractDataRef(object.designator);
+  std::optional<evaluate::DataRef> dataRef = ExtractDataRef(object.ref());
   assert(dataRef.has_value() &&
          "DataRef could not be extracted during mapping of derived type "
          "cannot proceed");
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index d20f9187640f0c0db8063ae1e2e25d6f0bfacfb4..0b4fe9044bfa7b1b62b07584d023150f88704a6b 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -17,6 +17,7 @@
 
 extern llvm::cl::opt<bool> treatIndexAsSection;
 extern llvm::cl::opt<bool> enableDelayedPrivatization;
+extern llvm::cl::opt<bool> enableDelayedPrivatizationStaging;
 
 namespace fir {
 class FirOpBuilder;
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 31e2c4082b32cae18a5832d29c73e8e5581e78cc..8d0ae2f195178c826a37a504c2e6065bcd3a07d1 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -882,10 +882,15 @@ static fir::ExtendedValue translateVariableToExtendedValue(
     mlir::Location loc, fir::FirOpBuilder &builder, hlfir::Entity variable,
     bool forceHlfirBase = false, bool contiguousHint = false) {
   assert(variable.isVariable() && "must be a variable");
-  /// When going towards FIR, use the original base value to avoid
-  /// introducing descriptors at runtime when they are not required.
-  mlir::Value base =
-      forceHlfirBase ? variable.getBase() : variable.getFirBase();
+  // When going towards FIR, use the original base value to avoid
+  // introducing descriptors at runtime when they are not required.
+  // This is not done for assumed-rank since the fir::ExtendedValue cannot
+  // held the related lower bounds in an vector. The lower bounds of the
+  // descriptor must always be used instead.
+
+  mlir::Value base = (forceHlfirBase || variable.isAssumedRank())
+                         ? variable.getBase()
+                         : variable.getFirBase();
   if (variable.isMutableBox())
     return fir::MutableBoxValue(base, getExplicitTypeParams(variable),
                                 fir::MutableProperties{});
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 1cd3976d0afbee7cfcc1ff53f8119a22a4d85efb..b3e1ee3da3a7786dc5133e946e3a604eead0f7bd 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -280,6 +280,10 @@ static constexpr IntrinsicHandler handlers[]{
        {"trim_name", asAddr, handleDynamicOptional},
        {"errmsg", asBox, handleDynamicOptional}}},
      /*isElemental=*/false},
+    {"getcwd",
+     &I::genGetCwd,
+     {{{"c", asBox}, {"status", asAddr, handleDynamicOptional}}},
+     /*isElemental=*/false},
     {"getpid", &I::genGetPID},
     {"iachar", &I::genIchar},
     {"iall",
@@ -3476,6 +3480,37 @@ mlir::Value IntrinsicLibrary::genFraction(mlir::Type resultType,
       fir::runtime::genFraction(builder, loc, fir::getBase(args[0])));
 }
 
+// GETCWD
+fir::ExtendedValue
+IntrinsicLibrary::genGetCwd(std::optional<mlir::Type> resultType,
+                            llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert((args.size() == 1 && resultType.has_value()) ||
+         (args.size() >= 1 && !resultType.has_value()));
+
+  mlir::Value cwd = fir::getBase(args[0]);
+  mlir::Value statusValue = fir::runtime::genGetCwd(builder, loc, cwd);
+
+  if (resultType.has_value()) {
+    // Function form, return status.
+    return statusValue;
+  } else {
+    // Subroutine form, store status and return none.
+    const fir::ExtendedValue &status = args[1];
+    if (!isStaticallyAbsent(status)) {
+      mlir::Value statusAddr = fir::getBase(status);
+      mlir::Value statusIsPresentAtRuntime =
+          builder.genIsNotNullAddr(loc, statusAddr);
+      builder.genIfThen(loc, statusIsPresentAtRuntime)
+          .genThen([&]() {
+            builder.createStoreWithConvert(loc, statusValue, statusAddr);
+          })
+          .end();
+    }
+  }
+
+  return {};
+}
+
 // GET_COMMAND
 void IntrinsicLibrary::genGetCommand(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 4);
@@ -4965,10 +5000,6 @@ fir::ExtendedValue
 IntrinsicLibrary::genIsContiguous(mlir::Type resultType,
                                   llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
-  if (const auto *boxValue = args[0].getBoxOf<fir::BoxValue>())
-    if (boxValue->hasAssumedRank())
-      TODO(loc, "intrinsic: is_contiguous with assumed rank argument");
-
   return builder.createConvert(
       loc, resultType,
       fir::runtime::genIsContiguous(builder, loc, fir::getBase(args[0])));
@@ -5961,15 +5992,45 @@ mlir::Value IntrinsicLibrary::genSetExponent(mlir::Type resultType,
                                    fir::getBase(args[1])));
 }
 
+/// Generate runtime call to inquire about all the bounds/extents of an
+/// assumed-rank array.
+template <typename Func>
+static fir::ExtendedValue genAssumedRankBoundInquiry(
+    fir::FirOpBuilder &builder, mlir::Location loc, mlir::Type resultType,
+    llvm::ArrayRef<fir::ExtendedValue> args, int kindPos, Func genRtCall) {
+  const fir::ExtendedValue &array = args[0];
+  // Allocate an array with the maximum rank, that is big enough to hold the
+  // result but still "small" (15 elements). Static size alloca make stack
+  // analysis/manipulation easier.
+  mlir::Type resultElementType = fir::unwrapSequenceType(resultType);
+  mlir::Type allocSeqType =
+      fir::SequenceType::get({Fortran::common::maxRank}, resultElementType);
+  mlir::Value resultStorage = builder.createTemporary(loc, allocSeqType);
+  mlir::Value arrayBox = builder.createBox(loc, array);
+  mlir::Value kind = isStaticallyAbsent(args, kindPos)
+                         ? builder.createIntegerConstant(
+                               loc, builder.getI32Type(),
+                               builder.getKindMap().defaultIntegerKind())
+                         : fir::getBase(args[kindPos]);
+  genRtCall(builder, loc, resultStorage, arrayBox, kind);
+  mlir::Type baseType =
+      fir::ReferenceType::get(builder.getVarLenSeqTy(resultElementType));
+  mlir::Value resultBase = builder.createConvert(loc, baseType, resultStorage);
+  mlir::Value rank =
+      builder.create<fir::BoxRankOp>(loc, builder.getIndexType(), arrayBox);
+  return fir::ArrayBoxValue{resultBase, {rank}};
+}
+
 // SHAPE
 fir::ExtendedValue
 IntrinsicLibrary::genShape(mlir::Type resultType,
                            llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() >= 1);
   const fir::ExtendedValue &array = args[0];
+  if (array.hasAssumedRank())
+    return genAssumedRankBoundInquiry(builder, loc, resultType, args,
+                                      /*kindPos=*/1, fir::runtime::genShape);
   int rank = array.rank();
-  if (rank == 0)
-    TODO(loc, "shape intrinsic lowering with assumed-rank source");
   mlir::Type indexType = builder.getIndexType();
   mlir::Type extentType = fir::unwrapSequenceType(resultType);
   mlir::Type seqType = fir::SequenceType::get(
@@ -6090,9 +6151,6 @@ IntrinsicLibrary::genSize(mlir::Type resultType,
   // Note that the value of the KIND argument is already reflected in the
   // resultType
   assert(args.size() == 3);
-  if (const auto *boxValue = args[0].getBoxOf<fir::BoxValue>())
-    if (boxValue->hasAssumedRank())
-      TODO(loc, "intrinsic: size with assumed rank argument");
 
   // Get the ARRAY argument
   mlir::Value array = builder.createBox(loc, args[0]);
@@ -6106,13 +6164,15 @@ IntrinsicLibrary::genSize(mlir::Type resultType,
 
   // Get the DIM argument.
   mlir::Value dim = fir::getBase(args[1]);
-  if (std::optional<std::int64_t> cstDim = fir::getIntIfConstant(dim)) {
-    // If it is a compile time constant, skip the runtime call.
-    return builder.createConvert(loc, resultType,
-                                 fir::factory::readExtent(builder, loc,
-                                                          fir::BoxValue{array},
-                                                          cstDim.value() - 1));
-  }
+  if (!args[0].hasAssumedRank())
+    if (std::optional<std::int64_t> cstDim = fir::getIntIfConstant(dim)) {
+      // If both DIM and the rank are compile time constants, skip the runtime
+      // call.
+      return builder.createConvert(
+          loc, resultType,
+          fir::factory::readExtent(builder, loc, fir::BoxValue{array},
+                                   cstDim.value() - 1));
+    }
   if (!fir::isa_ref_type(dim.getType()))
     return builder.createConvert(
         loc, resultType, fir::runtime::genSizeDim(builder, loc, array, dim));
diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index 76b920dba8697975f0df1c2db9d077e34037cf7b..16e543fe86a790f7993e3804dffb75041db9ac2f 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -394,6 +394,8 @@ static bool readToBoxValue(const fir::MutableBoxValue &box,
   // Track value as fir.box
   if ((box.isDerived() && mayBePolymorphic) || box.isUnlimitedPolymorphic())
     return true;
+  if (box.hasAssumedRank())
+    return true;
   // Intrinsic allocatables are contiguous, no need to track the value by
   // fir.box.
   if (box.isAllocatable() || box.rank() == 0)
@@ -409,14 +411,12 @@ fir::factory::genMutableBoxRead(fir::FirOpBuilder &builder, mlir::Location loc,
                                 const fir::MutableBoxValue &box,
                                 bool mayBePolymorphic,
                                 bool preserveLowerBounds) {
-  if (box.hasAssumedRank())
-    TODO(loc, "assumed rank allocatables or pointers");
   llvm::SmallVector<mlir::Value> lbounds;
   llvm::SmallVector<mlir::Value> extents;
   llvm::SmallVector<mlir::Value> lengths;
   if (readToBoxValue(box, mayBePolymorphic)) {
     auto reader = MutablePropertyReader(builder, loc, box);
-    if (preserveLowerBounds)
+    if (preserveLowerBounds && !box.hasAssumedRank())
       reader.getLowerBounds(lbounds);
     return fir::BoxValue{reader.getIrBox(), lbounds,
                          box.nonDeferredLenParams()};
diff --git a/flang/lib/Optimizer/Builder/Runtime/Command.cpp b/flang/lib/Optimizer/Builder/Runtime/Command.cpp
index 1d719e7bbd9a2d18dd03126dac1e5d72550e924f..8320d89493b3362a485cd7bbaaed8e42a6c888c1 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Command.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Command.cpp
@@ -88,3 +88,16 @@ mlir::Value fir::runtime::genGetEnvVariable(fir::FirOpBuilder &builder,
       sourceFile, sourceLine);
   return builder.create<fir::CallOp>(loc, runtimeFunc, args).getResult(0);
 }
+
+mlir::Value fir::runtime::genGetCwd(fir::FirOpBuilder &builder,
+                                    mlir::Location loc, mlir::Value cwd) {
+  mlir::func::FuncOp func =
+      fir::runtime::getRuntimeFunc<mkRTKey(GetCwd)>(loc, builder);
+  auto runtimeFuncTy = func.getFunctionType();
+  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+  mlir::Value sourceLine =
+      fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(2));
+  llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
+      builder, loc, runtimeFuncTy, cwd, sourceFile, sourceLine);
+  return builder.create<fir::CallOp>(loc, func, args).getResult(0);
+}
diff --git a/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp b/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
index 16f63bea4617a6bac907ffa5e5af8ff082f06f8b..34c4020b5907c9ba91db41d01368e00e386d6106 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Inquiry.cpp
@@ -87,3 +87,17 @@ mlir::Value fir::runtime::genIsContiguous(fir::FirOpBuilder &builder,
   auto args = fir::runtime::createArguments(builder, loc, fTy, array);
   return builder.create<fir::CallOp>(loc, isContiguousFunc, args).getResult(0);
 }
+
+void fir::runtime::genShape(fir::FirOpBuilder &builder, mlir::Location loc,
+                            mlir::Value resultAddr, mlir::Value array,
+                            mlir::Value kind) {
+  mlir::func::FuncOp func =
+      fir::runtime::getRuntimeFunc<mkRTKey(Shape)>(loc, builder);
+  auto fTy = func.getFunctionType();
+  auto sourceFile = fir::factory::locationToFilename(builder, loc);
+  auto sourceLine =
+      fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
+  auto args = fir::runtime::createArguments(
+      builder, loc, fTy, resultAddr, array, kind, sourceFile, sourceLine);
+  builder.create<fir::CallOp>(loc, func, args).getResult(0);
+}
diff --git a/flang/lib/Optimizer/Dialect/FIRAttr.cpp b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
index 2faba63dfba0739582dc9414b0a6ca8e6c8892fa..a0202a015922894ba6b08a922f1f46c8c14f15d2 100644
--- a/flang/lib/Optimizer/Dialect/FIRAttr.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
@@ -297,6 +297,6 @@ void fir::printFirAttribute(FIROpsDialect *dialect, mlir::Attribute attr,
 
 void FIROpsDialect::registerAttributes() {
   addAttributes<ClosedIntervalAttr, ExactTypeAttr, FortranVariableFlagsAttr,
-                LowerBoundAttr, PointIntervalAttr, RealAttr, SubclassAttr,
-                UpperBoundAttr>();
+                LowerBoundAttr, PointIntervalAttr, RealAttr, ReduceAttr,
+                SubclassAttr, UpperBoundAttr>();
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index b530a9dc1bcc47ced88cc3e55e8c85ee5126dd42..75ca738211abef5d62474de83dbfe5d0672bb17e 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -2456,9 +2456,16 @@ void fir::DoLoopOp::build(mlir::OpBuilder &builder,
                           mlir::OperationState &result, mlir::Value lb,
                           mlir::Value ub, mlir::Value step, bool unordered,
                           bool finalCountValue, mlir::ValueRange iterArgs,
+                          mlir::ValueRange reduceOperands,
+                          llvm::ArrayRef<mlir::Attribute> reduceAttrs,
                           llvm::ArrayRef<mlir::NamedAttribute> attributes) {
   result.addOperands({lb, ub, step});
+  result.addOperands(reduceOperands);
   result.addOperands(iterArgs);
+  result.addAttribute(getOperandSegmentSizeAttr(),
+                      builder.getDenseI32ArrayAttr(
+                          {1, 1, 1, static_cast<int32_t>(reduceOperands.size()),
+                           static_cast<int32_t>(iterArgs.size())}));
   if (finalCountValue) {
     result.addTypes(builder.getIndexType());
     result.addAttribute(getFinalValueAttrName(result.name),
@@ -2477,6 +2484,9 @@ void fir::DoLoopOp::build(mlir::OpBuilder &builder,
   if (unordered)
     result.addAttribute(getUnorderedAttrName(result.name),
                         builder.getUnitAttr());
+  if (!reduceAttrs.empty())
+    result.addAttribute(getReduceAttrsAttrName(result.name),
+                        builder.getArrayAttr(reduceAttrs));
   result.addAttributes(attributes);
 }
 
@@ -2502,24 +2512,51 @@ mlir::ParseResult fir::DoLoopOp::parse(mlir::OpAsmParser &parser,
   if (mlir::succeeded(parser.parseOptionalKeyword("unordered")))
     result.addAttribute("unordered", builder.getUnitAttr());
 
+  // Parse the reduction arguments.
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> reduceOperands;
+  llvm::SmallVector<mlir::Type> reduceArgTypes;
+  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
+    // Parse reduction attributes and variables.
+    llvm::SmallVector<ReduceAttr> attributes;
+    if (failed(parser.parseCommaSeparatedList(
+            mlir::AsmParser::Delimiter::Paren, [&]() {
+              if (parser.parseAttribute(attributes.emplace_back()) ||
+                  parser.parseArrow() ||
+                  parser.parseOperand(reduceOperands.emplace_back()) ||
+                  parser.parseColonType(reduceArgTypes.emplace_back()))
+                return mlir::failure();
+              return mlir::success();
+            })))
+      return mlir::failure();
+    // Resolve input operands.
+    for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes))
+      if (parser.resolveOperand(std::get<0>(operand_type),
+                                std::get<1>(operand_type), result.operands))
+        return mlir::failure();
+    llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
+                                                 attributes.end());
+    result.addAttribute(getReduceAttrsAttrName(result.name),
+                        builder.getArrayAttr(arrayAttr));
+  }
+
   // Parse the optional initial iteration arguments.
   llvm::SmallVector<mlir::OpAsmParser::Argument> regionArgs;
-  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> iterOperands;
   llvm::SmallVector<mlir::Type> argTypes;
   bool prependCount = false;
   regionArgs.push_back(inductionVariable);
 
   if (succeeded(parser.parseOptionalKeyword("iter_args"))) {
     // Parse assignment list and results type list.
-    if (parser.parseAssignmentList(regionArgs, operands) ||
+    if (parser.parseAssignmentList(regionArgs, iterOperands) ||
         parser.parseArrowTypeList(result.types))
       return mlir::failure();
-    if (result.types.size() == operands.size() + 1)
+    if (result.types.size() == iterOperands.size() + 1)
       prependCount = true;
     // Resolve input operands.
     llvm::ArrayRef<mlir::Type> resTypes = result.types;
-    for (auto operand_type :
-         llvm::zip(operands, prependCount ? resTypes.drop_front() : resTypes))
+    for (auto operand_type : llvm::zip(
+             iterOperands, prependCount ? resTypes.drop_front() : resTypes))
       if (parser.resolveOperand(std::get<0>(operand_type),
                                 std::get<1>(operand_type), result.operands))
         return mlir::failure();
@@ -2530,6 +2567,12 @@ mlir::ParseResult fir::DoLoopOp::parse(mlir::OpAsmParser &parser,
     prependCount = true;
   }
 
+  // Set the operandSegmentSizes attribute
+  result.addAttribute(getOperandSegmentSizeAttr(),
+                      builder.getDenseI32ArrayAttr(
+                          {1, 1, 1, static_cast<int32_t>(reduceOperands.size()),
+                           static_cast<int32_t>(iterOperands.size())}));
+
   if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
     return mlir::failure();
 
@@ -2606,6 +2649,10 @@ mlir::LogicalResult fir::DoLoopOp::verify() {
 
     i++;
   }
+  auto reduceAttrs = getReduceAttrsAttr();
+  if (getNumReduceOperands() != (reduceAttrs ? reduceAttrs.size() : 0))
+    return emitOpError(
+        "mismatch in number of reduction variables and reduction attributes");
   return mlir::success();
 }
 
@@ -2615,6 +2662,17 @@ void fir::DoLoopOp::print(mlir::OpAsmPrinter &p) {
     << getUpperBound() << " step " << getStep();
   if (getUnordered())
     p << " unordered";
+  if (hasReduceOperands()) {
+    p << " reduce(";
+    auto attrs = getReduceAttrsAttr();
+    auto operands = getReduceOperands();
+    llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) {
+      p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
+        << std::get<1>(it).getType();
+    });
+    p << ')';
+    printBlockTerminators = true;
+  }
   if (hasIterOperands()) {
     p << " iter_args(";
     auto regionArgs = getRegionIterArgs();
@@ -2628,8 +2686,9 @@ void fir::DoLoopOp::print(mlir::OpAsmPrinter &p) {
     p << " -> " << getResultTypes();
     printBlockTerminators = true;
   }
-  p.printOptionalAttrDictWithKeyword((*this)->getAttrs(),
-                                     {"unordered", "finalValue"});
+  p.printOptionalAttrDictWithKeyword(
+      (*this)->getAttrs(),
+      {"unordered", "finalValue", "reduceAttrs", "operandSegmentSizes"});
   p << ' ';
   p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                 printBlockTerminators);
diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index b265c74c33dd5eb13b90cefd369da58f579846da..648628fd1c9af05a32ef320839f7619a2384c740 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -45,17 +45,11 @@ namespace {
 class ExternalNameConversionPass
     : public fir::impl::ExternalNameConversionBase<ExternalNameConversionPass> {
 public:
-  ExternalNameConversionPass(bool appendUnderscoring)
-      : appendUnderscores(appendUnderscoring) {}
-
-  ExternalNameConversionPass() { usePassOpt = true; }
+  using ExternalNameConversionBase<
+      ExternalNameConversionPass>::ExternalNameConversionBase;
 
   mlir::ModuleOp getModule() { return getOperation(); }
   void runOnOperation() override;
-
-private:
-  bool appendUnderscores;
-  bool usePassOpt = false;
 };
 } // namespace
 
@@ -63,7 +57,6 @@ void ExternalNameConversionPass::runOnOperation() {
   auto op = getOperation();
   auto *context = &getContext();
 
-  appendUnderscores = (usePassOpt) ? appendUnderscoreOpt : appendUnderscores;
   llvm::DenseMap<mlir::StringAttr, mlir::FlatSymbolRefAttr> remappings;
   // Update names of external Fortran functions and names of Common Block
   // globals.
@@ -74,7 +67,8 @@ void ExternalNameConversionPass::runOnOperation() {
           mlir::SymbolTable::getSymbolAttrName());
       auto deconstructedName = fir::NameUniquer::deconstruct(symName);
       if (fir::NameUniquer::isExternalFacingUniquedName(deconstructedName)) {
-        auto newName = mangleExternalName(deconstructedName, appendUnderscores);
+        auto newName =
+            mangleExternalName(deconstructedName, appendUnderscoreOpt);
         auto newAttr = mlir::StringAttr::get(context, newName);
         mlir::SymbolTable::setSymbolName(&funcOrGlobal, newAttr);
         auto newSymRef = mlir::FlatSymbolRefAttr::get(newAttr);
@@ -101,12 +95,3 @@ void ExternalNameConversionPass::runOnOperation() {
       nestedOp->setAttr(update.first, update.second);
   });
 }
-
-std::unique_ptr<mlir::Pass> fir::createExternalNameConversionPass() {
-  return std::make_unique<ExternalNameConversionPass>();
-}
-
-std::unique_ptr<mlir::Pass>
-fir::createExternalNameConversionPass(bool appendUnderscoring) {
-  return std::make_unique<ExternalNameConversionPass>(appendUnderscoring);
-}
diff --git a/flang/lib/Optimizer/Transforms/FunctionAttr.cpp b/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
index f54080f21b9675afa0f50df596942ae685a3f1df..69c41595974ef9d0c4b297ab82f5572a9409a213 100644
--- a/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
+++ b/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 
 namespace fir {
-#define GEN_PASS_DECL_FUNCTIONATTR
 #define GEN_PASS_DEF_FUNCTIONATTR
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
@@ -76,22 +75,3 @@ void FunctionAttrPass::runOnOperation() {
 
   LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");
 }
-
-std::unique_ptr<mlir::Pass> fir::createFunctionAttrPass(
-    fir::FunctionAttrTypes &functionAttr, bool noInfsFPMath, bool noNaNsFPMath,
-    bool approxFuncFPMath, bool noSignedZerosFPMath, bool unsafeFPMath) {
-  FunctionAttrOptions opts;
-  // Frame pointer
-  opts.framePointerKind = functionAttr.framePointerKind;
-  opts.noInfsFPMath = noInfsFPMath;
-  opts.noNaNsFPMath = noNaNsFPMath;
-  opts.approxFuncFPMath = approxFuncFPMath;
-  opts.noSignedZerosFPMath = noSignedZerosFPMath;
-  opts.unsafeFPMath = unsafeFPMath;
-
-  return std::make_unique<FunctionAttrPass>(opts);
-}
-
-std::unique_ptr<mlir::Pass> fir::createFunctionAttrPass() {
-  return std::make_unique<FunctionAttrPass>();
-}
diff --git a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
index 601a937de37be9b3ba963f556d63f6b6f02d1214..d311167c58b4d63f949b6ee2fb369d9bb33eb948 100644
--- a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
+++ b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp
@@ -38,7 +38,6 @@
 #include <algorithm>
 
 namespace fir {
-#define GEN_PASS_DECL_VSCALEATTR
 #define GEN_PASS_DEF_VSCALEATTR
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
@@ -77,14 +76,3 @@ void VScaleAttrPass::runOnOperation() {
 
   LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");
 }
-
-std::unique_ptr<mlir::Pass>
-fir::createVScaleAttrPass(std::pair<unsigned, unsigned> vscaleAttr) {
-  VScaleAttrOptions opts;
-  opts.vscaleRange = vscaleAttr;
-  return std::make_unique<VScaleAttrPass>(opts);
-}
-
-std::unique_ptr<mlir::Pass> fir::createVScaleAttrPass() {
-  return std::make_unique<VScaleAttrPass>();
-}
diff --git a/flang/runtime/command.cpp b/flang/runtime/command.cpp
index b573c5dfd797b373235a9b5428d5ad11070ffcca..e642248a25e6880141312196879f3a462c76498a 100644
--- a/flang/runtime/command.cpp
+++ b/flang/runtime/command.cpp
@@ -17,12 +17,19 @@
 
 #ifdef _WIN32
 #include "flang/Common/windows-include.h"
+#include <direct.h>
+#define getcwd _getcwd
+#define PATH_MAX MAX_PATH
 
 // On Windows GetCurrentProcessId returns a DWORD aka uint32_t
 #include <processthreadsapi.h>
 inline pid_t getpid() { return GetCurrentProcessId(); }
 #else
 #include <unistd.h> //getpid()
+
+#ifndef PATH_MAX
+#define PATH_MAX 4096
+#endif
 #endif
 
 namespace Fortran::runtime {
@@ -239,4 +246,23 @@ std::int32_t RTNAME(GetEnvVariable)(const Descriptor &name,
   return StatOk;
 }
 
+std::int32_t RTNAME(GetCwd)(
+    const Descriptor &cwd, const char *sourceFile, int line) {
+  Terminator terminator{sourceFile, line};
+
+  RUNTIME_CHECK(terminator, IsValidCharDescriptor(&cwd));
+
+  char *buf{(char *)AllocateMemoryOrCrash(terminator, PATH_MAX)};
+
+  if (!getcwd(buf, PATH_MAX)) {
+    return StatMissingCurrentWorkDirectory;
+  }
+
+  std::int64_t strLen{StringLength(buf)};
+  std::int32_t status{CopyCharsToDescriptor(cwd, buf, strLen)};
+
+  std::free(buf);
+  return status;
+}
+
 } // namespace Fortran::runtime
diff --git a/flang/runtime/inquiry.cpp b/flang/runtime/inquiry.cpp
index 2b59a1cfab1a9b0dc87690c73e4865251899c2d4..5ffd9755fd0e52d67c3116621b0665829d125fd0 100644
--- a/flang/runtime/inquiry.cpp
+++ b/flang/runtime/inquiry.cpp
@@ -18,6 +18,15 @@
 
 namespace Fortran::runtime {
 
+template <int KIND> struct RawStoreIntegerAt {
+  RT_API_ATTRS void operator()(
+      void *contiguousIntegerArray, std::size_t at, std::int64_t value) const {
+    reinterpret_cast<Fortran::runtime::CppTypeFor<
+        Fortran::common::TypeCategory::Integer, KIND> *>(
+        contiguousIntegerArray)[at] = value;
+  }
+};
+
 extern "C" {
 std::int64_t RTDEF(LboundDim)(
     const Descriptor &array, int dim, const char *sourceFile, int line) {
@@ -76,5 +85,27 @@ std::int64_t RTDEF(SizeDim)(
   return static_cast<std::int64_t>(dimension.Extent());
 }
 
+void RTDEF(Shape)(void *result, const Descriptor &array, int kind,
+    const char *sourceFile, int line) {
+  Terminator terminator{sourceFile, line};
+  INTERNAL_CHECK(array.rank() <= common::maxRank);
+  for (SubscriptValue i{0}; i < array.rank(); ++i) {
+    const Dimension &dimension{array.GetDimension(i)};
+    Fortran::runtime::ApplyIntegerKind<RawStoreIntegerAt, void>(
+        kind, terminator, result, i, dimension.Extent());
+  }
+}
+
+void RTDEF(Lbound)(void *result, const Descriptor &array, int kind,
+    const char *sourceFile, int line) {
+  Terminator terminator{sourceFile, line};
+  INTERNAL_CHECK(array.rank() <= common::maxRank);
+  for (SubscriptValue i{0}; i < array.rank(); ++i) {
+    const Dimension &dimension{array.GetDimension(i)};
+    Fortran::runtime::ApplyIntegerKind<RawStoreIntegerAt, void>(
+        kind, terminator, result, i, dimension.LowerBound());
+  }
+}
+
 } // extern "C"
 } // namespace Fortran::runtime
diff --git a/flang/runtime/stat.h b/flang/runtime/stat.h
index 4f46f52ecb29415ef1007bf7e93e5660acded48d..71faeb027d9085fc8ef2c1ed4525bc7c6e3b2a24 100644
--- a/flang/runtime/stat.h
+++ b/flang/runtime/stat.h
@@ -41,6 +41,7 @@ enum Stat {
   StatLocked = FORTRAN_RUNTIME_STAT_LOCKED,
   StatLockedOtherImage = FORTRAN_RUNTIME_STAT_LOCKED_OTHER_IMAGE,
   StatMissingEnvVariable = FORTRAN_RUNTIME_STAT_MISSING_ENV_VAR,
+  StatMissingCurrentWorkDirectory = FORTRAN_RUNTIME_STAT_MISSING_CWD,
   StatStoppedImage = FORTRAN_RUNTIME_STAT_STOPPED_IMAGE,
   StatUnlocked = FORTRAN_RUNTIME_STAT_UNLOCKED,
   StatUnlockedFailedImage = FORTRAN_RUNTIME_STAT_UNLOCKED_FAILED_IMAGE,
diff --git a/flang/test/Fir/loop03.fir b/flang/test/Fir/loop03.fir
new file mode 100644
index 0000000000000000000000000000000000000000..b88dcaf8639be553371b5440ca4d32562b643b88
--- /dev/null
+++ b/flang/test/Fir/loop03.fir
@@ -0,0 +1,17 @@
+// Test the reduction semantics of fir.do_loop
+// RUN: fir-opt %s | FileCheck %s
+
+func.func @reduction() {
+  %bound = arith.constant 10 : index
+  %step = arith.constant 1 : index
+  %sum = fir.alloca i32
+// CHECK: %[[VAL_0:.*]] = fir.alloca i32
+// CHECK: fir.do_loop %[[VAL_1:.*]] = %[[VAL_2:.*]] to %[[VAL_3:.*]] step %[[VAL_4:.*]] unordered reduce(#fir.reduce_attr<add> -> %[[VAL_0]] : !fir.ref<i32>) {
+  fir.do_loop %iv = %step to %bound step %step unordered reduce(#fir.reduce_attr<add> -> %sum : !fir.ref<i32>) {
+    %index = fir.convert %iv : (index) -> i32
+    %1 = fir.load %sum : !fir.ref<i32>
+    %2 = arith.addi %index, %1 : i32
+    fir.store %2 to %sum : !fir.ref<i32>
+  }
+  return
+}
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90
new file mode 100644
index 0000000000000000000000000000000000000000..6c8f5ba48d1bf9043f52a725313b8a373b7c91a0
--- /dev/null
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90
@@ -0,0 +1,108 @@
+! Test lowering of SIZE/SIZEOF inquiry intrinsics with assumed-ranks
+! arguments.
+! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+
+
+subroutine test_size_1(x)
+  real :: x(..)
+  call takes_integer(size(x))
+end subroutine
+
+subroutine test_size_2(x)
+  real :: x(..)
+  call takes_integer(size(x, 2))
+end subroutine
+
+subroutine test_size_3(x, d)
+  real :: x(..)
+  integer, optional :: d
+  call takes_integer(size(x, d))
+end subroutine
+
+subroutine test_size_4(x)
+  real, allocatable :: x(..)
+  call takes_integer(size(x))
+end subroutine
+
+
+! CHECK-LABEL:   func.func @_QPtest_size_1(
+! CHECK-SAME:                              %[[VAL_0:.*]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_size_1Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:           %[[VAL_7:.*]] = fir.call @_FortranASize(%[[VAL_5]]
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i64) -> i32
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_size_2(
+! CHECK-SAME:                              %[[VAL_0:.*]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.alloca i32
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_size_2Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+! CHECK:           fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref<i32>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<i32>) -> i64
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
+! CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (i32) {
+! CHECK:             %[[VAL_11:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:             %[[VAL_13:.*]] = fir.call @_FortranASize(%[[VAL_11]]
+! CHECK:             %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i64) -> i32
+! CHECK:             fir.result %[[VAL_14]] : i32
+! CHECK:           } else {
+! CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_18:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:             %[[VAL_20:.*]] = fir.call @_FortranASizeDim(%[[VAL_18]]
+! CHECK:             %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i64) -> i32
+! CHECK:             fir.result %[[VAL_21]] : i32
+! CHECK:           }
+! CHECK:           %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_22]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_22]]#1, %[[VAL_22]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_size_3(
+! CHECK-SAME:                              %[[VAL_0:.*]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"},
+! CHECK-SAME:                              %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "d", fir.optional}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_size_3Ed"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_size_3Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.ref<i32>) -> i64
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
+! CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (i32) {
+! CHECK:             %[[VAL_11:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:             %[[VAL_13:.*]] = fir.call @_FortranASize(%[[VAL_11]],
+! CHECK:             %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i64) -> i32
+! CHECK:             fir.result %[[VAL_14]] : i32
+! CHECK:           } else {
+! CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<i32>
+! CHECK:             %[[VAL_18:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:             %[[VAL_20:.*]] = fir.call @_FortranASizeDim(%[[VAL_18]]
+! CHECK:             %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i64) -> i32
+! CHECK:             fir.result %[[VAL_21]] : i32
+! CHECK:           }
+! CHECK:           %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_22]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_22]]#1, %[[VAL_22]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_size_4(
+! CHECK-SAME:                              %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_size_4Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<*:f32>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.call @_FortranASize(%[[VAL_6]]
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i64) -> i32
+! CHECK:           %[[VAL_10:.*]]:3 = hlfir.associate %[[VAL_9]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_10]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_10]]#1, %[[VAL_10]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90
new file mode 100644
index 0000000000000000000000000000000000000000..bbeff5ff051915f51730c57a6206e277a2793e0e
--- /dev/null
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90
@@ -0,0 +1,56 @@
+! Test shape lowering for assumed-rank
+! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+
+subroutine test_shape(x)
+  real :: x(..)
+  call takes_integer_array(shape(x))
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_shape(
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<15xi32>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 4 : i32
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi32>>) -> !fir.llvm_ptr<i8>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_3:.*]] : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:           %[[VAL_10:.*]] = fir.call @_FortranAShape(%[[VAL_7]], %[[VAL_8]], %[[VAL_4]], %{{.*}}, %{{.*}})
+! CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK:           %[[VAL_12:.*]] = fir.box_rank %[[VAL_3]] : (!fir.box<!fir.array<*:f32>>) -> index
+! CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_11]](%[[VAL_13]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_15:.*]] = arith.constant false
+! CHECK:           %[[VAL_16:.*]] = hlfir.as_expr %[[VAL_14]]#0 move %[[VAL_15]] : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<?xi32>
+! CHECK:           %[[VAL_17:.*]]:3 = hlfir.associate %[[VAL_16]](%[[VAL_13]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+! CHECK:           fir.call @_QPtakes_integer_array(%[[VAL_17]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xi32>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_17]]#1, %[[VAL_17]]#2 : !fir.ref<!fir.array<?xi32>>, i1
+! CHECK:           hlfir.destroy %[[VAL_16]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+subroutine test_shape_kind(x)
+  real :: x(..)
+  call takes_integer8_array(shape(x, kind=8))
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_shape_kind(
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<15xi64>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 8 : i32
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi64>>) -> !fir.llvm_ptr<i8>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_3:.*]] : (!fir.box<!fir.array<*:f32>>) -> !fir.box<none>
+! CHECK:           %[[VAL_10:.*]] = fir.call @_FortranAShape(%[[VAL_7]], %[[VAL_8]], %[[VAL_4]], %{{.*}}, %{{.*}})
+! CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi64>>) -> !fir.ref<!fir.array<?xi64>>
+! CHECK:           %[[VAL_12:.*]] = fir.box_rank %[[VAL_3]] : (!fir.box<!fir.array<*:f32>>) -> index
+! CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_11]](%[[VAL_13]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.array<?xi64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi64>>, !fir.ref<!fir.array<?xi64>>)
+
+subroutine test_shape_2(x)
+  real, pointer :: x(..)
+  call takes_integer_array(shape(x))
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_shape_2(
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<15xi32>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_5:.*]] = arith.constant 4 : i32
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi32>>) -> !fir.llvm_ptr<i8>
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_11:.*]] = fir.call @_FortranAShape(%[[VAL_8]], %[[VAL_9]], %[[VAL_5]], %{{.*}}, %{{.*}})
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<15xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK:           %[[VAL_13:.*]] = fir.box_rank %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> index
+! CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_13]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_14]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90
new file mode 100644
index 0000000000000000000000000000000000000000..a1d150a21d1497f40678c97f7e75ae3f7491a5f3
--- /dev/null
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90
@@ -0,0 +1,383 @@
+! Test lowering of inquiry intrinsics with assumed-ranks arguments.
+! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+
+subroutine test_allocated(x)
+  real, allocatable :: x(..)
+  call takes_logical(allocated(x))
+end subroutine
+
+subroutine test_associated_1(x)
+  real, pointer :: x(..)
+  call takes_logical(associated(x))
+end subroutine
+
+subroutine test_associated_2(x, y)
+  real, pointer :: x(..)
+  real, target :: y(:)
+  call takes_logical(associated(x, y))
+end subroutine
+
+subroutine test_associated_3(x, y)
+  real, pointer :: x(..)
+  real, pointer :: y(..)
+  call takes_logical(associated(x, y))
+end subroutine
+
+subroutine test_len_1(x)
+  character(*) :: x(..)
+  call takes_integer(len(x))
+end subroutine
+
+subroutine test_len_2(x)
+  character(*), pointer :: x(..)
+  call takes_integer(len(x))
+end subroutine
+
+subroutine test_storage_size_1(x)
+  class(*) :: x(..)
+  call takes_integer(storage_size(x))
+end subroutine
+
+subroutine test_storage_size_2(x)
+  class(*), pointer :: x(..)
+  call takes_integer(storage_size(x))
+end subroutine
+
+subroutine test_present_1(x)
+  class(*), optional :: x(..)
+  call takes_logical(present(x))
+end subroutine
+
+subroutine test_present_2(x)
+  class(*), optional, pointer :: x(..)
+  call takes_logical(present(x))
+end subroutine
+
+subroutine test_is_contiguous_1(x)
+  class(*) :: x(..)
+  call takes_logical(is_contiguous(x))
+end subroutine
+
+subroutine test_is_contiguous_2(x)
+  class(*), pointer :: x(..)
+  call takes_logical(is_contiguous(x))
+end subroutine
+
+subroutine test_same_type_as_1(x, y)
+  class(*) :: x(..), y(..)
+  call takes_logical(same_type_as(x, y))
+end subroutine
+
+subroutine test_same_type_as_2(x, y)
+  class(*), pointer :: x(..), y(..)
+  call takes_logical(same_type_as(x, y))
+end subroutine
+
+subroutine test_extends_type_of_1(x, y)
+  class(*) :: x(..), y(..)
+  call takes_logical(extends_type_of(x, y))
+end subroutine
+
+subroutine test_extends_type_of_2(x, y)
+  class(*), pointer :: x(..), y(..)
+  call takes_logical(extends_type_of(x, y))
+end subroutine
+
+subroutine c_loc_1(x)
+  use iso_c_binding, only : c_loc
+  real, target :: x(..)
+  call takes_cloc(c_loc(x))
+end subroutine
+
+subroutine c_loc_2(x)
+  use iso_c_binding, only : c_loc
+  real, pointer :: x(..)
+  call takes_cloc(c_loc(x))
+end subroutine
+
+! CHECK-LABEL:   func.func @_QPtest_allocated(
+! CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_allocatedEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<*:f32>>>) -> !fir.heap<!fir.array<*:f32>>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.heap<!fir.array<*:f32>>) -> i64
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_5]], %[[VAL_6]] : i64
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_associated_1(
+! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_associated_1Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.ptr<!fir.array<*:f32>>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ptr<!fir.array<*:f32>>) -> i64
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_5]], %[[VAL_6]] : i64
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_associated_2(
+! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                    %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "y", fir.target}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_associated_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtest_associated_2Ey"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.call @_FortranAPointerIsAssociatedWith(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_10:.*]]:3 = hlfir.associate %[[VAL_9]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_10]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_10]]#1, %[[VAL_10]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_associated_3(
+! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                    %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_associated_3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_associated_3Ey"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranAPointerIsAssociatedWith(%[[VAL_7]], %[[VAL_8]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_len_1(
+! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.box<!fir.array<*:!fir.char<1,?>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_len_1Ex"} : (!fir.box<!fir.array<*:!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<*:!fir.char<1,?>>>, !fir.box<!fir.array<*:!fir.char<1,?>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]]#0 : (!fir.box<!fir.array<*:!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (index) -> i32
+! CHECK:           %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_5]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_len_2(
+! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>) -> index
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_3]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_len_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:!fir.char<1,?>>>>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_3]] : (index) -> i32
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_6]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_storage_size_1(
+! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_storage_size_1Ex"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]]#0 : (!fir.class<!fir.array<*:none>>) -> i32
+! CHECK:           %[[VAL_4:.*]] = arith.constant 8 : i32
+! CHECK:           %[[VAL_5:.*]] = arith.muli %[[VAL_3]], %[[VAL_4]] : i32
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_6]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_storage_size_2(
+! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_storage_size_2Ex"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.ptr<!fir.array<*:none>>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ptr<!fir.array<*:none>>) -> i64
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64
+! CHECK:           fir.if %[[VAL_7]] {
+! CHECK:             %[[VAL_13:.*]] = fir.call @_FortranAReportFatalUserError
+! CHECK:           }
+! CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_15:.*]] = fir.box_elesize %[[VAL_14]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> i32
+! CHECK:           %[[VAL_16:.*]] = arith.constant 8 : i32
+! CHECK:           %[[VAL_17:.*]] = arith.muli %[[VAL_15]], %[[VAL_16]] : i32
+! CHECK:           %[[VAL_18:.*]]:3 = hlfir.associate %[[VAL_17]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           fir.call @_QPtakes_integer(%[[VAL_18]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_18]]#1, %[[VAL_18]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_present_1(
+! CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "x", fir.optional}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_present_1Ex"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_3:.*]] = fir.is_present %[[VAL_2]]#0 : (!fir.class<!fir.array<*:none>>) -> i1
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_5]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_present_2(
+! CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "x", fir.optional}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<optional, pointer>, uniq_name = "_QFtest_present_2Ex"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.is_present %[[VAL_2]]#0 : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>) -> i1
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_5]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_is_contiguous_1(
+! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_is_contiguous_1Ex"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.class<!fir.array<*:none>>) -> !fir.box<none>
+! CHECK:           %[[VAL_4:.*]] = fir.call @_FortranAIsContiguous(%[[VAL_3]]) fastmath<contract> : (!fir.box<none>) -> i1
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_6]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_is_contiguous_2(
+! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_is_contiguous_2Ex"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_5:.*]] = fir.call @_FortranAIsContiguous(%[[VAL_4]]) fastmath<contract> : (!fir.box<none>) -> i1
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_7:.*]]:3 = hlfir.associate %[[VAL_6]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_7]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_same_type_as_1(
+! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_same_type_as_1Ex"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_same_type_as_1Ey"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.class<!fir.array<*:none>>) -> !fir.box<none>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.class<!fir.array<*:none>>) -> !fir.box<none>
+! CHECK:           %[[VAL_7:.*]] = fir.call @_FortranASameTypeAs(%[[VAL_5]], %[[VAL_6]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_same_type_as_2(
+! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_same_type_as_2Ex"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_same_type_as_2Ey"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_5]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranASameTypeAs(%[[VAL_7]], %[[VAL_8]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_extends_type_of_1(
+! CHECK-SAME:                                         %[[VAL_0:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                         %[[VAL_1:.*]]: !fir.class<!fir.array<*:none>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_extends_type_of_1Ex"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_extends_type_of_1Ey"} : (!fir.class<!fir.array<*:none>>, !fir.dscope) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.class<!fir.array<*:none>>) -> !fir.box<none>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.class<!fir.array<*:none>>) -> !fir.box<none>
+! CHECK:           %[[VAL_7:.*]] = fir.call @_FortranAExtendsTypeOf(%[[VAL_5]], %[[VAL_6]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_extends_type_of_2(
+! CHECK-SAME:                                         %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                         %[[VAL_1:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_extends_type_of_2Ex"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_extends_type_of_2Ey"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>)
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<*:none>>>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_5]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.class<!fir.ptr<!fir.array<*:none>>>) -> !fir.box<none>
+! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranAExtendsTypeOf(%[[VAL_7]], %[[VAL_8]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>) -> i1
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPc_loc_1(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x", fir.target}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFc_loc_1Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_5:.*]] = fir.coordinate_of %[[VAL_3]], %[[VAL_4]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+! CHECK:           %[[VAL_6:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.box<!fir.array<*:f32>>) -> !fir.ref<!fir.array<*:f32>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.array<*:f32>>) -> i64
+! CHECK:           fir.store %[[VAL_7]] to %[[VAL_5]] : !fir.ref<i64>
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK:           %[[VAL_9:.*]] = arith.constant false
+! CHECK:           %[[VAL_10:.*]] = hlfir.as_expr %[[VAL_8]]#0 move %[[VAL_9]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1) -> !hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>
+! CHECK:           %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1)
+! CHECK:           fir.call @_QPtakes_cloc(%[[VAL_11]]#1) fastmath<contract> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1
+! CHECK:           hlfir.destroy %[[VAL_10]] : !hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPc_loc_2(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFc_loc_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_5:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_4]], %[[VAL_5]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+! CHECK:           %[[VAL_7:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box<!fir.ptr<!fir.array<*:f32>>>) -> !fir.ptr<!fir.array<*:f32>>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.ptr<!fir.array<*:f32>>) -> i64
+! CHECK:           fir.store %[[VAL_8]] to %[[VAL_6]] : !fir.ref<i64>
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK:           %[[VAL_10:.*]] = arith.constant false
+! CHECK:           %[[VAL_11:.*]] = hlfir.as_expr %[[VAL_9]]#0 move %[[VAL_10]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1) -> !hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>
+! CHECK:           %[[VAL_12:.*]]:3 = hlfir.associate %[[VAL_11]] {adapt.valuebyref} : (!hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1)
+! CHECK:           fir.call @_QPtakes_cloc(%[[VAL_12]]#1) fastmath<contract> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_12]]#1, %[[VAL_12]]#2 : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1
+! CHECK:           hlfir.destroy %[[VAL_11]] : !hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>
+! CHECK:           return
+! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90 b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
index cd65696ca5ede919ab33f858bb82bfc7843d6b9e..3b60b0d04e9fa3b4ae38b59a1999f65786381cac 100644
--- a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
+++ b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
@@ -103,7 +103,7 @@ end subroutine
 ! CHECK-SAME:                                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable, intent_out>, uniq_name = "_QMassumed_rank_testsFtest_intentoutEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
 ! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<*:f32>>>) -> !fir.heap<!fir.array<*:f32>>
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.heap<!fir.array<*:f32>>) -> i64
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
@@ -111,7 +111,7 @@ end subroutine
 ! CHECK:           fir.if %[[VAL_7]] {
 ! CHECK:             %[[VAL_8:.*]] = arith.constant false
 ! CHECK:             %[[VAL_9:.*]] = fir.absent !fir.box<none>
-! CHECK:             %[[VAL_12:.*]] = fir.convert %[[VAL_2]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> !fir.ref<!fir.box<none>>
+! CHECK:             %[[VAL_12:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK:             %[[VAL_14:.*]] = fir.call @_FortranAAllocatableDeallocate(%[[VAL_12]], %[[VAL_8]], %[[VAL_9]], %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           }
 ! CHECK:           return
diff --git a/flang/test/Lower/HLFIR/select-rank.f90 b/flang/test/Lower/HLFIR/select-rank.f90
index 84987531f1419e8e480796d88d3e9ff57c3723a0..211b7565bab8a36af010539094af66fd12208525 100644
--- a/flang/test/Lower/HLFIR/select-rank.f90
+++ b/flang/test/Lower/HLFIR/select-rank.f90
@@ -319,7 +319,7 @@ end subroutine
 ! CHECK:           %[[VAL_6:.*]] = fir.is_assumed_size %[[VAL_2]]#0 : (!fir.box<!fir.array<*:f32>>) -> i1
 ! CHECK:           cf.cond_br %[[VAL_6]], ^bb1, ^bb2
 ! CHECK:         ^bb1:
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {uniq_name = "_QFtest_simple_caseEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {uniq_name = "_QFtest_simple_caseEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_7]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb2:
@@ -362,7 +362,7 @@ end subroutine
 ! CHECK:           fir.call @_QPr2(%[[VAL_8]]#0) fastmath<contract> : (!fir.box<!fir.array<?x?xf32>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {uniq_name = "_QFtest_rank_starEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {uniq_name = "_QFtest_rank_starEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_9]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb4:
@@ -431,7 +431,7 @@ end subroutine
 ! CHECK:           fir.call @_QPr2(%[[VAL_7]]#0) fastmath<contract> : (!fir.box<!fir.array<?x?xf32>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {fortran_attrs = #fir.var_attrs<asynchronous, target>, uniq_name = "_QFtest_rank_star_attributesEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {fortran_attrs = #fir.var_attrs<asynchronous, target>, uniq_name = "_QFtest_rank_star_attributesEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_8]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb4:
@@ -469,7 +469,7 @@ end subroutine
 ! CHECK:           fir.call @_QPr2_implicit(%[[VAL_14]]#1) fastmath<contract> : (!fir.ref<!fir.array<?x?xf32>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtest_rank_star_contiguousEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtest_rank_star_contiguousEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_15]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb4:
@@ -518,7 +518,7 @@ end subroutine
 ! CHECK:           fir.call @_QPrc0_implicit(%[[VAL_16]]#0) fastmath<contract> : (!fir.boxchar<1>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_8]]#1 typeparams %[[VAL_7]] {uniq_name = "_QFtest_rank_star_contiguous_characterEx"} : (!fir.box<!fir.array<*:!fir.char<1,?>>>, i64) -> (!fir.box<!fir.array<*:!fir.char<1,?>>>, !fir.box<!fir.array<*:!fir.char<1,?>>>)
+! CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_8]]#0 typeparams %[[VAL_7]] {uniq_name = "_QFtest_rank_star_contiguous_characterEx"} : (!fir.box<!fir.array<*:!fir.char<1,?>>>, i64) -> (!fir.box<!fir.array<*:!fir.char<1,?>>>, !fir.box<!fir.array<*:!fir.char<1,?>>>)
 ! CHECK:           fir.call @_QPrcdefault(%[[VAL_17]]#0) fastmath<contract> : (!fir.box<!fir.array<*:!fir.char<1,?>>>) -> ()
 ! CHECK:           cf.br ^bb6
 ! CHECK:         ^bb4:
@@ -568,7 +568,7 @@ end subroutine
 ! CHECK:           fir.call @_QPra0(%[[VAL_10]]#0) fastmath<contract> : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_simple_allocEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_simple_allocEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
 ! CHECK:           fir.call @_QPradefault(%[[VAL_11]]#0) fastmath<contract> : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb4:
@@ -588,7 +588,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = fir.box_rank %[[VAL_2]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> i8
 ! CHECK:           fir.select_case %[[VAL_4]] : i8 [#fir.point, %[[VAL_3]], ^bb2, unit, ^bb1]
 ! CHECK:         ^bb1:
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_character_allocEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_character_allocEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
 ! CHECK:           cf.br ^bb3
 ! CHECK:         ^bb2:
 ! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
@@ -614,7 +614,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = fir.box_rank %[[VAL_8]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> i8
 ! CHECK:           fir.select_case %[[VAL_10]] : i8 [#fir.point, %[[VAL_9]], ^bb2, unit, ^bb1]
 ! CHECK:         ^bb1:
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_8]]#1 typeparams %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_explicit_character_ptrEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, i64) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_8]]#0 typeparams %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_explicit_character_ptrEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, i64) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
 ! CHECK:           cf.br ^bb3
 ! CHECK:         ^bb2:
 ! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_8]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
@@ -634,7 +634,7 @@ end subroutine
 ! CHECK:           %[[VAL_6:.*]] = fir.box_rank %[[VAL_4]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> i8
 ! CHECK:           fir.select_case %[[VAL_6]] : i8 [#fir.point, %[[VAL_5]], ^bb2, unit, ^bb1]
 ! CHECK:         ^bb1:
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_4]]#1 typeparams %[[VAL_3]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_assumed_character_ptrEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_3]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_assumed_character_ptrEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>)
 ! CHECK:           cf.br ^bb3
 ! CHECK:         ^bb2:
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:!fir.char<1,?>>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
@@ -661,7 +661,7 @@ end subroutine
 ! CHECK:           fir.call @_QPrup1(%[[VAL_8]]#0) fastmath<contract> : (!fir.class<!fir.array<?xnone>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb3:
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {uniq_name = "_QFtest_polymorphicEx"} : (!fir.class<!fir.array<*:none>>) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {uniq_name = "_QFtest_polymorphicEx"} : (!fir.class<!fir.array<*:none>>) -> (!fir.class<!fir.array<*:none>>, !fir.class<!fir.array<*:none>>)
 ! CHECK:           fir.call @_QPrupdefault(%[[VAL_9]]#0) fastmath<contract> : (!fir.class<!fir.array<*:none>>) -> ()
 ! CHECK:           cf.br ^bb5
 ! CHECK:         ^bb4:
@@ -711,7 +711,7 @@ end subroutine
 ! CHECK:           fir.call @_QPr1(%[[VAL_20]]#0) fastmath<contract> : (!fir.box<!fir.array<?xf32>>) -> ()
 ! CHECK:           cf.br ^bb7
 ! CHECK:         ^bb6:
-! CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_4]]#1 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_4]]#0 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPr0(%[[VAL_11]]#1) fastmath<contract> : (!fir.ref<f32>) -> ()
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_21]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb7
@@ -741,14 +741,14 @@ end subroutine
 ! CHECK:           fir.call @_QPr1(%[[VAL_32]]#0) fastmath<contract> : (!fir.box<!fir.array<?xf32>>) -> ()
 ! CHECK:           cf.br ^bb13
 ! CHECK:         ^bb12:
-! CHECK:           %[[VAL_33:.*]]:2 = hlfir.declare %[[VAL_4]]#1 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_33:.*]]:2 = hlfir.declare %[[VAL_4]]#0 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPr1(%[[VAL_23]]#0) fastmath<contract> : (!fir.box<!fir.array<?xf32>>) -> ()
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_33]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb13
 ! CHECK:         ^bb13:
 ! CHECK:           cf.br ^bb20
 ! CHECK:         ^bb14:
-! CHECK:           %[[VAL_34:.*]]:2 = hlfir.declare %[[VAL_3]]#1 {uniq_name = "_QFtest_nested_select_rankEx1"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_34:.*]]:2 = hlfir.declare %[[VAL_3]]#0 {uniq_name = "_QFtest_nested_select_rankEx1"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           %[[VAL_35:.*]] = arith.constant 0 : i8
 ! CHECK:           %[[VAL_36:.*]] = arith.constant 1 : i8
 ! CHECK:           %[[VAL_37:.*]] = fir.is_assumed_size %[[VAL_4]]#0 : (!fir.box<!fir.array<*:f32>>) -> i1
@@ -770,7 +770,7 @@ end subroutine
 ! CHECK:           fir.call @_QPr1(%[[VAL_43]]#0) fastmath<contract> : (!fir.box<!fir.array<?xf32>>) -> ()
 ! CHECK:           cf.br ^bb19
 ! CHECK:         ^bb18:
-! CHECK:           %[[VAL_44:.*]]:2 = hlfir.declare %[[VAL_4]]#1 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_44:.*]]:2 = hlfir.declare %[[VAL_4]]#0 {uniq_name = "_QFtest_nested_select_rankEx2"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_34]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           fir.call @_QPrdefault(%[[VAL_44]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
 ! CHECK:           cf.br ^bb19
@@ -789,7 +789,7 @@ end subroutine
 ! CHECK:           %[[VAL_5:.*]] = fir.is_assumed_size %[[VAL_2]]#0 : (!fir.box<!fir.array<*:f32>>) -> i1
 ! CHECK:           cf.cond_br %[[VAL_5]], ^bb1, ^bb2
 ! CHECK:         ^bb1:
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {uniq_name = "_QFtest_branchingEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]]#0 {uniq_name = "_QFtest_branchingEx"} : (!fir.box<!fir.array<*:f32>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
 ! CHECK:           %[[VAL_7:.*]] = fir.call @_QPjump() fastmath<contract> : () -> !fir.logical<4>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.logical<4>) -> i1
 ! CHECK:           %[[VAL_9:.*]] = arith.constant true
diff --git a/flang/test/Lower/Intrinsics/getcwd-function.f90 b/flang/test/Lower/Intrinsics/getcwd-function.f90
new file mode 100644
index 0000000000000000000000000000000000000000..50b64729294fe3be8af5aa8237e08613025b43e3
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/getcwd-function.f90
@@ -0,0 +1,23 @@
+! Test GETCWD with dynamically optional arguments.
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPtest(
+! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}) -> i32 {
+integer function test(cwd)
+  CHARACTER(len=255) :: cwd
+  test = getcwd(cwd)
+  ! CHECK-NEXT:        %[[c8:.*]] = arith.constant 8 : i32
+  ! CHECK-NEXT:        %[[c255:.*]] = arith.constant 255 : index
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  ! CHECK-NEXT:        %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtestEcwd"} : (!fir.ref<!fir.char<1,255>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[test:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
+  ! CHECK-NEXT:        %[[testAddr:.*]] = fir.declare %[[test]] {uniq_name = "_QFtestEtest"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  ! CHECK-NEXT:        %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref<!fir.char<1,255>>) -> !fir.box<!fir.char<1,255>>
+  ! CHECK:             %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box<!fir.char<1,255>>) -> !fir.box<none>
+  ! CHECK:             %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_9:.*]], %[[c8]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> i32
+  ! CHECK-NEXT:        fir.store %[[statusValue]] to %[[testAddr]] : !fir.ref<i32>
+  ! CHECK-NEXT:        %[[returnValue:.*]] = fir.load %[[testAddr]] : !fir.ref<i32>
+  ! CHECK-NEXT:        return %[[returnValue]] : i32
+end function
diff --git a/flang/test/Lower/Intrinsics/getcwd-optional.f90 b/flang/test/Lower/Intrinsics/getcwd-optional.f90
new file mode 100644
index 0000000000000000000000000000000000000000..3e2a221f0c3f97cade42ee4d6bd3d5f9fcb75925
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/getcwd-optional.f90
@@ -0,0 +1,29 @@
+! Test GETCWD with dynamically optional arguments.
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+
+! CHECK-LABEL: func.func @_QPtest(
+! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"},
+! CHECK-SAME: %[[statusArg:.*]]: !fir.ref<i32> {fir.bindc_name = "status", fir.optional}) {
+subroutine test(cwd, status)
+  CHARACTER(len=255) :: cwd
+  INTEGER, OPTIONAL :: status
+  call getcwd(cwd, status)
+  ! CHECK-NEXT:        %[[c0:.*]] = arith.constant 0 : i64
+  ! CHECK-NEXT:        %[[c11:.*]] = arith.constant 11 : i32
+  ! CHECK-NEXT:        %[[c255:.*]] = arith.constant 255 : index
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  ! CHECK-NEXT:        %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtestEcwd"} : (!fir.ref<!fir.char<1,255>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[statusAddr:.*]] = fir.declare %[[statusArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtestEstatus"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  ! CHECK-NEXT:       %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref<!fir.char<1,255>>) -> !fir.box<!fir.char<1,255>>
+  ! CHECK:            %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box<!fir.char<1,255>>) -> !fir.box<none>
+  ! CHECK:            %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_8:.*]], %[[c11]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> i32
+  ! CHECK-NEXT:       %[[statusCast:.*]] = fir.convert %[[statusAddr]] : (!fir.ref<i32>) -> i64
+  ! CHECK-NEXT:       %[[isPresent:.*]] = arith.cmpi ne, %[[statusCast]], %[[c0]] : i64
+  ! CHECK-NEXT:       fir.if %[[isPresent]] {
+  ! CHECK-NEXT:         fir.store %[[statusValue]] to %[[statusAddr]] : !fir.ref<i32>
+  ! CHECK-NEXT:       }
+  ! CHECK-NEXT:       return
+end subroutine
diff --git a/flang/test/Lower/Intrinsics/getcwd.f90 b/flang/test/Lower/Intrinsics/getcwd.f90
new file mode 100644
index 0000000000000000000000000000000000000000..fe207854aff0aa062bea57d47d9fba55ed02b784
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/getcwd.f90
@@ -0,0 +1,44 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPcwd_only(
+! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}) {
+subroutine cwd_only(cwd)
+  CHARACTER(len=255) :: cwd
+  call getcwd(cwd)
+  ! CHECK-NEXT:        %[[c7:.*]] = arith.constant 7 : i32
+  ! CHECK-NEXT:        %[[c255:.*]] = arith.constant 255 : index
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  ! CHECK-NEXT:        %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFcwd_onlyEcwd"} : (!fir.ref<!fir.char<1,255>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref<!fir.char<1,255>>) -> !fir.box<!fir.char<1,255>>
+  ! CHECK:             %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box<!fir.char<1,255>>) -> !fir.box<none>
+  ! CHECK:             %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_7:.*]], %[[c7]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> i32
+  ! CHECK-NEXT:        return
+end subroutine cwd_only
+
+! CHECK-LABEL: func.func @_QPall_arguments(
+! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"},
+! CHECK-SAME: %[[statusArg:.*]]: !fir.ref<i32> {fir.bindc_name = "status"}) {
+subroutine all_arguments(cwd, status)
+  CHARACTER(len=255) :: cwd
+  INTEGER :: status
+  call getcwd(cwd, status)
+  ! CHECK-NEXT:        %[[c0:.*]] = arith.constant 0 : i64
+  ! CHECK-NEXT:        %[[c26:.*]] = arith.constant 26 : i32
+  ! CHECK-NEXT:        %[[c255:.*]] = arith.constant 255 : index
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  ! CHECK-NEXT:        %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argumentsEcwd"} : (!fir.ref<!fir.char<1,255>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,255>>
+  ! CHECK-NEXT:        %[[statusAddr:.*]] = fir.declare %[[statusArg]] dummy_scope %0 {uniq_name = "_QFall_argumentsEstatus"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  ! CHECK-NEXT:       %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref<!fir.char<1,255>>) -> !fir.box<!fir.char<1,255>>
+  ! CHECK:            %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box<!fir.char<1,255>>) -> !fir.box<none>
+  ! CHECK:            %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_8:.*]], %[[c26]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> i32
+  ! CHECK-NEXT:       %[[statusCast:.*]] = fir.convert %[[statusAddr]] : (!fir.ref<i32>) -> i64
+  ! CHECK-NEXT:       %[[isPresent:.*]] = arith.cmpi ne, %[[statusCast]], %[[c0]] : i64
+  ! CHECK-NEXT:       fir.if %[[isPresent]] {
+  ! CHECK-NEXT:         fir.store %[[statusValue]] to %[[statusAddr]] : !fir.ref<i32>
+  ! CHECK-NEXT:       }
+  ! CHECK-NEXT:       return
+end subroutine all_arguments
\ No newline at end of file
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
new file mode 100644
index 0000000000000000000000000000000000000000..a27de1152ce17a2c82ca965de45187d6556ca05a
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
@@ -0,0 +1,71 @@
+! Tests delayed privatization for `targets ... private(..)` for allocatables.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN:   | FileCheck %s
+
+subroutine target_allocatable
+  implicit none
+  integer, allocatable :: alloc_var
+
+  !$omp target private(alloc_var)
+    alloc_var = 10
+  !$omp end target
+end subroutine target_allocatable
+
+! CHECK-LABEL: omp.private {type = private}
+! CHECK-SAME:    @[[VAR_PRIVATIZER_SYM:.*]] :
+! CHECK-SAME:      [[TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+! CHECK:  ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
+! CHECK:    %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "alloc_var", {{.*}}}
+
+! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+
+! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
+! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}}
+! CHECK-NEXT:     %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   } else {
+! CHECK-NEXT:     %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap<i32>
+! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   }
+
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
+
+! CHECK-NEXT: } dealloc {
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
+
+! CHECK-NEXT:   %[[PRIV_VAL:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:   %[[PRIV_ADDR:.*]] = fir.box_addr %[[PRIV_VAL]]
+! CHECK-NEXT:   %[[PRIV_ADDR_I64:.*]] = fir.convert %[[PRIV_ADDR]]
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
+! CHECK-NEXT:   %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64
+
+! CHECK-NEXT:   fir.if %[[PRIV_NULL_COND]] {
+! CHECK:          %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:     %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]]
+! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR_2]]
+! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
+! CHECK-NEXT:     %[[ZEROS_BOX:.*]]  = fir.embox %[[ZEROS]]
+! CHECK-NEXT:     fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]]
+! CHECK-NEXT:   }
+
+! CHECK-NEXT:   omp.yield
+! CHECK-NEXT: }
+
+
+! CHECK-LABEL: func.func @_QPtarget_allocatable() {
+
+! CHECK:  %[[VAR_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
+! CHECK-SAME: {bindc_name = "alloc_var", {{.*}}}
+! CHECK:  %[[VAR_DECL:.*]]:2 = hlfir.declare %[[VAR_ALLOC]]
+
+! CHECK:  omp.target private(
+! CHECK-SAME: @[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %{{.*}} : [[TYPE]]) {
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
new file mode 100644
index 0000000000000000000000000000000000000000..6e8282b2af625533611af6e73437180aadf54271
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -0,0 +1,175 @@
+! Tests delayed privatization for `targets ... private(..)` for allocatables.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN:   | FileCheck %s
+
+subroutine target_allocatable(lb, ub, l)
+  implicit none
+  integer mapped_var
+  integer, allocatable :: alloc_var
+  real :: real_var
+
+  integer(8) :: lb, ub
+  real, dimension(lb:ub) :: real_arr
+
+  complex :: comp_var
+
+  integer(8):: l
+  character(len = l)  :: char_var
+
+  !$omp target private(alloc_var, real_var) private(lb, real_arr) &
+  !$omp&  private(comp_var) private(char_var)
+    mapped_var = 5
+
+    alloc_var = 10
+    real_var = 3.14
+
+    real_arr(lb + 1) = 6.28
+
+    comp_var = comp_var * comp_var
+
+    char_var = "hello"
+  !$omp end target
+end subroutine target_allocatable
+
+! Test the privatizer for `character`
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[CHAR_PRIVATIZER_SYM:[^[:space:]]+char_var[^[:space:]]+]]
+! CHECK-SAME:   : [[CHAR_TYPE:!fir.boxchar<1>]] alloc {
+!
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[CHAR_TYPE]]):
+! CHECK-NEXT:   %[[UNBOX:.*]]:2 = fir.unboxchar %[[PRIV_ARG]]
+! CHECK:        %[[PRIV_ALLOC:.*]] = fir.alloca !fir.char<1,?>(%[[UNBOX]]#1 : index)
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] typeparams %[[UNBOX]]#1
+! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[CHAR_TYPE]])
+! CHECK-NEXT: }
+
+! Test the privatizer for `complex`
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[COMP_PRIVATIZER_SYM:[^[:space:]]+comp_var[^[:space:]]+]]
+! CHECK-SAME:   : [[COMP_TYPE:!fir.ref<!fir.complex<4>>]] alloc {
+!
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[COMP_TYPE]]):
+! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.complex<4>
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[COMP_TYPE]])
+! CHECK-NEXT: }
+
+! Test the privatizer for `real(:)`
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[ARR_PRIVATIZER_SYM:[^[:space:]]+real_arr[^[:space:]]+]]
+! CHECK-SAME:   : [[ARR_TYPE:!fir.box<!fir.array<\?xf32>>]] alloc {
+!
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ARR_TYPE]]):
+! CHECK:        %[[C0:.*]] = arith.constant 0 : index
+! CHECK-NEXT:   %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]] : ([[ARR_TYPE]], index)
+! CHECK:        %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<{{\?}}xf32>
+! CHECK-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE_SHIFT]])
+! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[ARR_TYPE]])
+! CHECK-NEXT: }
+
+! Test the privatizer for `real(:)`'s lower bound
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[LB_PRIVATIZER_SYM:[^[:space:]]+lb[^[:space:]]+]]
+! CHECK-SAME:   : [[LB_TYPE:!fir.ref<i64>]] alloc {
+
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[LB_TYPE]]):
+! CHECK-NEXT:   %[[PRIV_ALLOCA:.*]] = fir.alloca i64
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]]
+! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[LB_TYPE]])
+! CHECK-NEXT: }
+
+! Test the privatizer for `real`
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[REAL_PRIVATIZER_SYM:[^[:space:]]+real_var[^[:space:]]+]]
+! CHECK-SAME:   : [[REAL_TYPE:!fir.ref<f32>]] alloc {
+
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[REAL_TYPE]]):
+! CHECK-NEXT:   %[[PRIV_ALLOCA:.*]] = fir.alloca f32
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]]
+! CHECK-NEXT:  omp.yield(%[[PRIV_DECL]]#0 : [[REAL_TYPE]])
+! CHECK-NEXT: }
+
+! Test the privatizer for `allocatable`
+!
+! CHECK:      omp.private {type = private}
+! CHECK-SAME:   @[[ALLOC_PRIVATIZER_SYM:[^[:space:]]+alloc_var[^[:space:]]+]]
+! CHECK-SAME:   : [[ALLOC_TYPE:!fir.ref<!fir.box<!fir.heap<i32>>>]] alloc {
+!
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]):
+! CHECK:        %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+! CHECK-NEXT:   %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap<i32>) -> i64
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
+! CHECK-NEXT:   %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64
+!
+! CHECK-NEXT:   fir.if %[[ALLOC_COND]] {
+! CHECK:          %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}}
+! CHECK-NEXT:     %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   } else {
+! CHECK-NEXT:     %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap<i32>
+! CHECK-NEXT:     %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK-NEXT:     fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT:   }
+!
+! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
+! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[ALLOC_TYPE]])
+!
+! CHECK-NEXT: } dealloc {
+! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]):
+!
+! CHECK-NEXT:   %[[PRIV_VAL:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:   %[[PRIV_ADDR:.*]] = fir.box_addr %[[PRIV_VAL]]
+! CHECK-NEXT:   %[[PRIV_ADDR_I64:.*]] = fir.convert %[[PRIV_ADDR]]
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : i64
+! CHECK-NEXT:   %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64
+!
+! CHECK-NEXT:   fir.if %[[PRIV_NULL_COND]] {
+! CHECK:          %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]]
+! CHECK-NEXT:     %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]]
+! CHECK-NEXT:     fir.freemem %[[PRIV_ADDR_2]]
+! CHECK-NEXT:     %[[ZEROS:.*]] = fir.zero_bits
+! CHECK-NEXT:     %[[ZEROS_BOX:.*]]  = fir.embox %[[ZEROS]]
+! CHECK-NEXT:     fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]]
+! CHECK-NEXT:   }
+!
+! CHECK-NEXT:   omp.yield
+! CHECK-NEXT: }
+
+! CHECK:      func.func @_QPtarget_allocatable
+! CHECK:        %[[MAPPED_ALLOC:.*]] = fir.alloca i32 {bindc_name = "mapped_var", {{.*}}}
+! CHECK-NEXT:   %[[MAPPED_DECL:.*]]:2 = hlfir.declare %[[MAPPED_ALLOC]]
+! CHECK:        %[[MAPPED_MI:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref<i32>, i32)
+
+! CHECK:        omp.target
+! CHECK-SAME:     map_entries(%[[MAPPED_MI]] -> %[[MAPPED_ARG:.*]] : !fir.ref<i32>)
+! CHECK-SAME:     private(
+! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>,
+! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:.*]] : !fir.ref<f32>,
+! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:.*]] : !fir.ref<i64>,
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:.*]] : !fir.box<!fir.array<?xf32>>,
+! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:.*]] : !fir.ref<!fir.complex<4>>,
+! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:.*]] : !fir.boxchar<1>) {
+! CHECK-NOT:      fir.alloca
+! CHECK:          hlfir.declare %[[MAPPED_ARG]]
+! CHECK:          hlfir.declare %[[ALLOC_ARG]]
+! CHECK:          hlfir.declare %[[REAL_ARG]]
+! CHECK:          hlfir.declare %[[LB_ARG]]
+! CHECK:          %[[ARR_ARG_ADDR:.*]] = fir.box_addr %[[ARR_ARG]]
+! CHECK:          hlfir.declare %[[ARR_ARG_ADDR]]
+! CHECK:          hlfir.declare %[[COMP_ARG]]
+! CHECK:          %[[CHAR_ARG_UNBOX:.*]]:2 = fir.unboxchar %[[CHAR_ARG]]
+! CHECK:          hlfir.declare %[[CHAR_ARG_UNBOX]]
+! CHECK:          omp.terminator
+! CHECK-NEXT:   }
+
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90
new file mode 100644
index 0000000000000000000000000000000000000000..524e973780c49fb3a8aef59ea9b4a9bd10ed7590
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90
@@ -0,0 +1,40 @@
+! Tests delayed privatization for `targets ... private(..)` for simple variables.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN:   | FileCheck %s
+
+subroutine target_simple
+  implicit none
+  integer :: simple_var
+
+  !$omp target private(simple_var)
+    simple_var = 10
+  !$omp end target
+end subroutine target_simple
+
+! CHECK-LABEL: omp.private {type = private}
+! CHECK-SAME:              @[[VAR_PRIVATIZER_SYM:.*]] : !fir.ref<i32> alloc {
+! CHECK:  ^bb0(%[[PRIV_ARG:.*]]: !fir.ref<i32>):
+! CHECK:    %[[PRIV_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}}
+! CHECK:    %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
+! CHECK:    omp.yield(%[[PRIV_DECL]]#0 : !fir.ref<i32>)
+! CHECK: }
+
+! CHECK-LABEL: func.func @_QPtarget_simple() {
+! CHECK:  %[[VAR_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}}
+! CHECK:  %[[VAR_DECL:.*]]:2 = hlfir.declare %[[VAR_ALLOC]]
+
+! CHECK:  omp.target private(
+! CHECK-SAME: @[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:    ^bb0(%[[REG_ARG:.*]]: !fir.ref<i32>):
+! CHECK:      %[[REG_DECL:.*]]:2 = hlfir.declare %[[REG_ARG]]
+! CHECK:      %[[C10:.*]] = arith.constant 10
+! CHECK:      hlfir.assign %[[C10]] to %[[REG_DECL]]#0
+! CHECK:      omp.terminator
+! CHECK:    }
+
+! CHECK:    return
+! CHECK:  }
+
diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90
index d62c58b3081addf0a214215c4e34b64de747cfb8..c52ae688811e967c14d1f0140b5c7cba3eeac179 100644
--- a/flang/test/Lower/OpenMP/critical.f90
+++ b/flang/test/Lower/OpenMP/critical.f90
@@ -51,3 +51,27 @@ subroutine predetermined_privatization()
   end do
   !$omp end parallel do
 end
+
+! https://github.com/llvm/llvm-project/issues/75767
+!CHECK-LABEL: func @_QPparallel_critical_privatization(
+subroutine parallel_critical_privatization()
+  integer :: i
+
+  !CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFparallel_critical_privatizationEi"}
+  !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFparallel_critical_privatizationEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !CHECK: omp.parallel {
+  !CHECK:   %[[PRIV_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFparallel_critical_privatizationEi"}
+  !CHECK:   %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFparallel_critical_privatizationEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !CHECK:   %[[TEMP:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref<i32>
+  !CHECK:   hlfir.assign %[[TEMP]] to %[[PRIV_I_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  !$omp parallel default(firstprivate)
+    !CHECK: omp.critical {
+    !$omp critical
+      !CHECK: %[[C200:.*]] = arith.constant 200 : i32
+      !CHECK: hlfir.assign %[[C200]] to %[[PRIV_I_DECL]]#0 : i32, !fir.ref<i32>
+      i = 200
+    !CHECK: }
+    !$omp end critical
+  !CHECK: }
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Lower/OpenMP/function-filtering-2.f90 b/flang/test/Lower/OpenMP/function-filtering-2.f90
index e1d0f72b9f525a934415adb6510db3cc1e301771..f367069efb3d9205b6c27205a85ca7b82dd373b0 100644
--- a/flang/test/Lower/OpenMP/function-filtering-2.f90
+++ b/flang/test/Lower/OpenMP/function-filtering-2.f90
@@ -1,6 +1,6 @@
-! RUN: %flang_fc1 -fopenmp -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefix=LLVM %s
+! RUN: %flang_fc1 -fopenmp -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-HOST %s
 ! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
-! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefix=LLVM %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-DEVICE %s
 ! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
 ! RUN: bbc -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
@@ -38,7 +38,7 @@ end subroutine no_declaretarget
 ! MLIR-ALL: return
 
 ! LLVM-HOST: define {{.*}} @{{.*}}main{{.*}}(
-! LLVM-HOST-NOT: {{.*}} @{{.*}}__omp_offloading{{.*}}main_{{.*}}(
+! LLVM-HOST: {{.*}} @{{.*}}__omp_offloading{{.*}}main_{{.*}}(
 ! LLVM-DEVICE-NOT: {{.*}} @{{.*}}main{{.*}}(
 ! LLVM-DEVICE: define {{.*}} @{{.*}}__omp_offloading{{.*}}main_{{.*}}(
 program main
diff --git a/flang/test/Lower/OpenMP/map-component-ref.f90 b/flang/test/Lower/OpenMP/map-component-ref.f90
index 2c582667f38d36ae205027ba9a324d289a823e5a..21b56ab303acd709caa9cf6ccef9e19f89aecba8 100644
--- a/flang/test/Lower/OpenMP/map-component-ref.f90
+++ b/flang/test/Lower/OpenMP/map-component-ref.f90
@@ -1,21 +1,22 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
-! CHECK: %[[V0:[0-9]+]] = fir.alloca !fir.type<_QFfooTt0{a0:i32,a1:i32}> {bindc_name = "a", uniq_name = "_QFfooEa"}
-! CHECK: %[[V1:[0-9]+]]:2 = hlfir.declare %[[V0]] {uniq_name = "_QFfooEa"} : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>)
-! CHECK: %[[V2:[0-9]+]] = hlfir.designate %[[V1]]#0{"a1"}   : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> !fir.ref<i32>
+! CHECK-LABEL: func.func @_QPfoo1
+! CHECK: %[[V0:[0-9]+]] = fir.alloca !fir.type<_QFfoo1Tt0{a0:i32,a1:i32}> {bindc_name = "a", uniq_name = "_QFfoo1Ea"}
+! CHECK: %[[V1:[0-9]+]]:2 = hlfir.declare %[[V0]] {uniq_name = "_QFfoo1Ea"} : (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>)
+! CHECK: %[[V2:[0-9]+]] = hlfir.designate %[[V1]]#0{"a1"}   : (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>) -> !fir.ref<i32>
 ! CHECK: %[[V3:[0-9]+]] = omp.map.info var_ptr(%[[V2]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "a%a1"}
-! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.type<_QFfooTt0{a0:i32,a1:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[V3]] : [1] : !fir.ref<i32>) -> !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>> {name = "a", partial_map = true}
-! CHECK: omp.target map_entries(%[[V3]] -> %arg0, %[[V4]] -> %arg1 : !fir.ref<i32>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) {
-! CHECK: ^bb0(%arg0: !fir.ref<i32>, %arg1: !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>):
-! CHECK:   %[[V5:[0-9]+]]:2 = hlfir.declare %arg1 {uniq_name = "_QFfooEa"} : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>)
+! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>, !fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[V3]] : [1] : !fir.ref<i32>) -> !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>> {name = "a", partial_map = true}
+! CHECK: omp.target map_entries(%[[V3]] -> %arg0, %[[V4]] -> %arg1 : !fir.ref<i32>, !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>) {
+! CHECK: ^bb0(%arg0: !fir.ref<i32>, %arg1: !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>):
+! CHECK:   %[[V5:[0-9]+]]:2 = hlfir.declare %arg1 {uniq_name = "_QFfoo1Ea"} : (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>)
 ! CHECK:   %c0_i32 = arith.constant 0 : i32
-! CHECK:   %[[V6:[0-9]+]] = hlfir.designate %[[V5]]#0{"a1"}   : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> !fir.ref<i32>
+! CHECK:   %[[V6:[0-9]+]] = hlfir.designate %[[V5]]#0{"a1"}   : (!fir.ref<!fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>>) -> !fir.ref<i32>
 ! CHECK:   hlfir.assign %c0_i32 to %[[V6]] : i32, !fir.ref<i32>
 ! CHECK:   omp.terminator
 ! CHECK: }
 
-subroutine foo()
+subroutine foo1()
   implicit none
 
   type t0
@@ -29,3 +30,25 @@ subroutine foo()
   !$omp end target
 end
 
+
+! CHECK-LABEL: func.func @_QPfoo2
+! CHECK-DAG: omp.map.info var_ptr(%{{[0-9]+}} : {{.*}} map_clauses(to) capture(ByRef) bounds(%{{[0-9]+}}) -> {{.*}} {name = "t%b(1_8)%a(1)"}
+! CHECK-DAG: omp.map.info var_ptr(%{{[0-9]+}} : {{.*}} map_clauses(from) capture(ByRef) bounds(%{{[0-9]+}}) -> {{.*}} {name = "u%b(1_8)%a(1)"}
+subroutine foo2()
+  implicit none
+
+  type t0
+    integer :: a(10)
+  end type
+
+  type t1
+    type(t0) :: b(10)
+  end type
+
+  type(t1) :: t, u
+
+!$omp target map(to: t%b(1)%a(1)) map(from: u%b(1)%a(1))
+  t%b(1)%a(1) = u%b(1)%a(1)
+!$omp end target
+
+end
diff --git a/flang/test/Semantics/getcwd.f90 b/flang/test/Semantics/getcwd.f90
new file mode 100644
index 0000000000000000000000000000000000000000..b6ff16eeec5ac7aa7f96c5a95fd0cf20bd0bbbf8
--- /dev/null
+++ b/flang/test/Semantics/getcwd.f90
@@ -0,0 +1,35 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+! Tests for the GETCWD intrinsics
+
+subroutine bad_kind_error(cwd, status)
+  CHARACTER(len=255) :: cwd
+  INTEGER(2) :: status
+  !ERROR: Actual argument for 'status=' has bad type or kind 'INTEGER(2)'
+  call getcwd(cwd, status)
+end subroutine bad_kind_error
+  
+subroutine bad_args_error()
+  !ERROR: missing mandatory 'c=' argument
+  call getcwd()
+end subroutine bad_args_error
+
+subroutine bad_apply_form(cwd)
+  CHARACTER(len=255) :: cwd
+  INTEGER :: status
+  !Declaration of 'getcwd'
+  call getcwd(cwd, status)
+  !ERROR: Cannot call subroutine 'getcwd' like a function
+  status = getcwd(cwd)
+end subroutine bad_apply_form
+
+subroutine good_subroutine(cwd, status)
+  CHARACTER(len=255) :: cwd
+  INTEGER :: status
+  call getcwd(cwd, status)
+end subroutine good_subroutine
+
+subroutine good_function(cwd, status)
+  CHARACTER(len=255) :: cwd
+  INTEGER :: status
+  status = getcwd(cwd)
+end subroutine good_function
\ No newline at end of file
diff --git a/flang/test/Transforms/debug-local-var-2.f90 b/flang/test/Transforms/debug-local-var-2.f90
index 0fe1b81c27e61e116953c3378242ba4ea02e86fb..ee60a07cc4bee697fa9816b3abfa9eada2f538bf 100644
--- a/flang/test/Transforms/debug-local-var-2.f90
+++ b/flang/test/Transforms/debug-local-var-2.f90
@@ -20,20 +20,20 @@
 
 ! CHECK-LABEL: define {{.*}}i64 @_QFPfn1
 ! CHECK-SAME: (ptr %[[ARG1:.*]], ptr %[[ARG2:.*]], ptr %[[ARG3:.*]])
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
 ! CHECK-DAG: %[[AL2:.*]] = alloca i64
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL2]], metadata ![[RES1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL2]], metadata ![[RES1:.*]], metadata !DIExpression())
 ! CHECK-LABEL: }
 
 ! CHECK-LABEL: define {{.*}}i32 @_QFPfn2
 ! CHECK-SAME: (ptr %[[FN2ARG1:.*]], ptr %[[FN2ARG2:.*]], ptr %[[FN2ARG3:.*]])
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
 ! CHECK-DAG: %[[AL3:.*]] = alloca i32
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL3]], metadata ![[RES2:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL3]], metadata ![[RES2:.*]], metadata !DIExpression())
 ! CHECK-LABEL: }
 
 program mn
diff --git a/flang/unittests/Runtime/Inquiry.cpp b/flang/unittests/Runtime/Inquiry.cpp
index 5b97bb239f49bd25e2f1d35fd54d73424a7f5469..98a350d3ad98a1f53557c647c6cbef7152420b33 100644
--- a/flang/unittests/Runtime/Inquiry.cpp
+++ b/flang/unittests/Runtime/Inquiry.cpp
@@ -14,7 +14,7 @@
 using namespace Fortran::runtime;
 using Fortran::common::TypeCategory;
 
-TEST(Inquiry, Lbound) {
+TEST(Inquiry, LboundDim) {
   // ARRAY  1 3 5
   //        2 4 6
   auto array{MakeArray<TypeCategory::Integer, 4>(
@@ -26,6 +26,42 @@ TEST(Inquiry, Lbound) {
   EXPECT_EQ(RTNAME(LboundDim)(*array, 2, __FILE__, __LINE__), std::int64_t{-1});
 }
 
+TEST(Inquiry, Lbound) {
+  // ARRAY  1 3 5
+  //        2 4 6
+  auto array{MakeArray<TypeCategory::Integer, 4>(
+      std::vector<int>{2, 3}, std::vector<std::int32_t>{1, 2, 3, 4, 5, 6})};
+  array->GetDimension(0).SetLowerBound(0);
+  array->GetDimension(1).SetLowerBound(-1);
+
+  // LBOUND(ARRAY, KIND=1)
+  auto int8Result{
+      MakeArray<TypeCategory::Integer, 1>(std::vector<int>{array->rank()},
+          std::vector<std::int8_t>(array->rank(), 0))};
+  RTNAME(Lbound)
+  (int8Result->raw().base_addr, *array, /*KIND=*/1, __FILE__, __LINE__);
+  EXPECT_EQ(*int8Result->ZeroBasedIndexedElement<std::int8_t>(0), 0);
+  EXPECT_EQ(*int8Result->ZeroBasedIndexedElement<std::int8_t>(1), -1);
+
+  // LBOUND(ARRAY, KIND=4)
+  auto int32Result{
+      MakeArray<TypeCategory::Integer, 4>(std::vector<int>{array->rank()},
+          std::vector<std::int32_t>(array->rank(), 0))};
+  RTNAME(Lbound)
+  (int32Result->raw().base_addr, *array, /*KIND=*/4, __FILE__, __LINE__);
+  EXPECT_EQ(*int32Result->ZeroBasedIndexedElement<std::int32_t>(0), 0);
+  EXPECT_EQ(*int32Result->ZeroBasedIndexedElement<std::int32_t>(1), -1);
+
+  // LBOUND(ARRAY, KIND=8)
+  auto int64Result{
+      MakeArray<TypeCategory::Integer, 8>(std::vector<int>{array->rank()},
+          std::vector<std::int64_t>(array->rank(), 0))};
+  RTNAME(Lbound)
+  (int64Result->raw().base_addr, *array, /*KIND=*/8, __FILE__, __LINE__);
+  EXPECT_EQ(*int64Result->ZeroBasedIndexedElement<std::int64_t>(0), 0);
+  EXPECT_EQ(*int64Result->ZeroBasedIndexedElement<std::int64_t>(1), -1);
+}
+
 TEST(Inquiry, Ubound) {
   // ARRAY  1 3 5
   //        2 4 6
@@ -76,3 +112,37 @@ TEST(Inquiry, Size) {
   EXPECT_EQ(RTNAME(SizeDim)(*array, 2, __FILE__, __LINE__), std::int64_t{3});
   EXPECT_EQ(RTNAME(Size)(*array, __FILE__, __LINE__), std::int64_t{6});
 }
+
+TEST(Inquiry, Shape) {
+  // ARRAY  1 3 5
+  //        2 4 6
+  auto array{MakeArray<TypeCategory::Integer, 4>(
+      std::vector<int>{2, 3}, std::vector<std::int32_t>{1, 2, 3, 4, 5, 6})};
+
+  // SHAPE(ARRAY, KIND=1)
+  auto int8Result{
+      MakeArray<TypeCategory::Integer, 1>(std::vector<int>{array->rank()},
+          std::vector<std::int8_t>(array->rank(), 0))};
+  RTNAME(Shape)
+  (int8Result->raw().base_addr, *array, /*KIND=*/1, __FILE__, __LINE__);
+  EXPECT_EQ(*int8Result->ZeroBasedIndexedElement<std::int8_t>(0), 2);
+  EXPECT_EQ(*int8Result->ZeroBasedIndexedElement<std::int8_t>(1), 3);
+
+  // SHAPE(ARRAY, KIND=4)
+  auto int32Result{
+      MakeArray<TypeCategory::Integer, 4>(std::vector<int>{array->rank()},
+          std::vector<std::int32_t>(array->rank(), 0))};
+  RTNAME(Shape)
+  (int32Result->raw().base_addr, *array, /*KIND=*/4, __FILE__, __LINE__);
+  EXPECT_EQ(*int32Result->ZeroBasedIndexedElement<std::int32_t>(0), 2);
+  EXPECT_EQ(*int32Result->ZeroBasedIndexedElement<std::int32_t>(1), 3);
+
+  // SHAPE(ARRAY, KIND=8)
+  auto int64Result{
+      MakeArray<TypeCategory::Integer, 8>(std::vector<int>{array->rank()},
+          std::vector<std::int64_t>(array->rank(), 0))};
+  RTNAME(Shape)
+  (int64Result->raw().base_addr, *array, /*KIND=*/8, __FILE__, __LINE__);
+  EXPECT_EQ(*int64Result->ZeroBasedIndexedElement<std::int64_t>(0), 2);
+  EXPECT_EQ(*int64Result->ZeroBasedIndexedElement<std::int64_t>(1), 3);
+}
diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 134c5143d6d6159efe0072766fc5995eab6b5be5..1f80e7f4e57c11bdb3e320d0ed092c2a1eca29c2 100644
--- a/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -44,6 +44,10 @@ function(create_object_library fq_target_name)
     message(FATAL_ERROR "'add_object_library' rule requires SRCS to be specified.")
   endif()
 
+  if(NOT ADD_OBJECT_CXX_STANDARD)
+    set(ADD_OBJECT_CXX_STANDARD ${CMAKE_CXX_STANDARD})
+  endif()
+
   set(internal_target_name ${fq_target_name}.__internal__)
   set(public_packaging_for_internal "-DLIBC_COPT_PUBLIC_PACKAGING")
 
@@ -75,6 +79,9 @@ function(create_object_library fq_target_name)
     target_include_directories(${internal_target_name} PRIVATE ${LIBC_SOURCE_DIR})
     target_compile_options(${internal_target_name} PRIVATE ${compile_options}
                            -fno-lto -march=${LIBC_GPU_TARGET_ARCHITECTURE})
+    set_target_properties(${internal_target_name}
+                          PROPERTIES
+                          CXX_STANDARD ${ADD_OBJECT_CXX_STANDARD})
   endif()
 
   if(SHOW_INTERMEDIATE_OBJECTS)
@@ -92,9 +99,6 @@ function(create_object_library fq_target_name)
     target_link_libraries(${fq_target_name} PUBLIC ${fq_deps_list})
   endif()
 
-  if(NOT ADD_OBJECT_CXX_STANDARD)
-    set(ADD_OBJECT_CXX_STANDARD ${CMAKE_CXX_STANDARD})
-  endif()
   set_target_properties(
     ${fq_target_name}
     PROPERTIES
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index b678350e9fcb15e3577d0ed89e91093dbc85be10..2217a696fc5d185dd308bbf38b7e4011e2073cc3 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -1,3 +1,13 @@
+if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
+  set(extra_entrypoints
+      # stdio.h entrypoints
+      libc.src.stdio.sprintf
+      libc.src.stdio.snprintf
+      libc.src.stdio.vsprintf
+      libc.src.stdio.vsnprintf
+  )
+endif()
+
 set(TARGET_LIBC_ENTRYPOINTS
     # assert.h entrypoints
     libc.src.assert.__assert_fail
@@ -175,6 +185,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.errno.errno
 
     # stdio.h entrypoints
+    ${extra_entrypoints}
     libc.src.stdio.feof
     libc.src.stdio.ferror
     libc.src.stdio.fseek
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index c990a5ba9e6695658180caca176d62e1330958e0..8863749e12c6db94a67358de1058c90d0b65b5fe 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -394,6 +394,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fminimum_mag_num
     libc.src.math.fminimum_mag_numf
     libc.src.math.fminimum_mag_numl
+    libc.src.math.fmul
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
@@ -505,13 +506,36 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fabsf16
     libc.src.math.fdimf16
     libc.src.math.floorf16
+    libc.src.math.fmaxf16
+    libc.src.math.fmaximumf16
+    libc.src.math.fmaximum_magf16
+    libc.src.math.fmaximum_mag_numf16
+    libc.src.math.fmaximum_numf16
+    libc.src.math.fminf16
+    libc.src.math.fminimumf16
+    libc.src.math.fminimum_magf16
+    libc.src.math.fminimum_mag_numf16
+    libc.src.math.fminimum_numf16
+    libc.src.math.frexpf16
     libc.src.math.fromfpf16
     libc.src.math.fromfpxf16
+    libc.src.math.ilogbf16
+    libc.src.math.llogbf16
     libc.src.math.llrintf16
     libc.src.math.llroundf16
+    libc.src.math.logbf16
     libc.src.math.lrintf16
     libc.src.math.lroundf16
+    # libc.src.math.modff16
+    libc.src.math.nanf16
     libc.src.math.nearbyintf16
+    libc.src.math.nextafterf16
+    libc.src.math.nextdownf16
+    # Temporarily disable nexttowardf16 on aarch64 because the conversion
+    # between _Float16 and long double will crash clang-11.  This is fixed in
+    # clang-12 and after: https://godbolt.org/z/8ceT9454c
+    # libc.src.math.nexttowardf16
+    libc.src.math.nextupf16
     libc.src.math.rintf16
     libc.src.math.roundf16
     libc.src.math.roundevenf16
@@ -557,6 +581,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.nextafterf128
     libc.src.math.nextdownf128
     libc.src.math.nextupf128
+    libc.src.math.remquof128
     libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.scalbnf128
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 335981ff7dc7cf1446796aeb6fc0f4b8a9f64212..d4f932416bd9f23e1dac8a3bec2882ec2b16501a 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -261,6 +261,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fminimum_mag_num
     libc.src.math.fminimum_mag_numf
     libc.src.math.fminimum_mag_numl
+    libc.src.math.fmul
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.frexp
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 479af40b5b26b99487ac7cbfeb05b6f0f8126556..67abf851b4d506efe087ca1a7aca790e5c26f77f 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -402,6 +402,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fminimum_mag_num
     libc.src.math.fminimum_mag_numf
     libc.src.math.fminimum_mag_numl
+    libc.src.math.fmul
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 780ffb6d972cda28085a6b3a4da65db98900ac65..31ad0bc412836c85f05b44917bbecaddc8b9db17 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -173,7 +173,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.atoll
     libc.src.stdlib.bsearch
     libc.src.stdlib.div
-    libc.src.stdlib.quick_exit
     libc.src.stdlib.labs
     libc.src.stdlib.ldiv
     libc.src.stdlib.llabs
@@ -422,6 +421,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fminimum_mag_num
     libc.src.math.fminimum_mag_numf
     libc.src.math.fminimum_mag_numl
+    libc.src.math.fmul
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
@@ -538,13 +538,34 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fabsf16
     libc.src.math.fdimf16
     libc.src.math.floorf16
+    libc.src.math.fmaxf16
+    libc.src.math.fmaximumf16
+    libc.src.math.fmaximum_magf16
+    libc.src.math.fmaximum_mag_numf16
+    libc.src.math.fmaximum_numf16
+    libc.src.math.fminf16
+    libc.src.math.fminimumf16
+    libc.src.math.fminimum_magf16
+    libc.src.math.fminimum_mag_numf16
+    libc.src.math.fminimum_numf16
+    libc.src.math.fmodf16
+    libc.src.math.frexpf16
     libc.src.math.fromfpf16
     libc.src.math.fromfpxf16
+    libc.src.math.ilogbf16
+    libc.src.math.llogbf16
     libc.src.math.llrintf16
     libc.src.math.llroundf16
+    libc.src.math.logbf16
     libc.src.math.lrintf16
     libc.src.math.lroundf16
+    libc.src.math.modff16
+    libc.src.math.nanf16
     libc.src.math.nearbyintf16
+    libc.src.math.nextafterf16
+    libc.src.math.nextdownf16
+    libc.src.math.nexttowardf16
+    libc.src.math.nextupf16
     libc.src.math.rintf16
     libc.src.math.roundf16
     libc.src.math.roundevenf16
@@ -591,6 +612,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.nextafterf128
     libc.src.math.nextdownf128
     libc.src.math.nextupf128
+    libc.src.math.remquof128
     libc.src.math.rintf128
     libc.src.math.roundevenf128
     libc.src.math.roundf128
@@ -756,9 +778,11 @@ if(LLVM_LIBC_FULL_BUILD)
     # stdlib.h entrypoints
     libc.src.stdlib._Exit
     libc.src.stdlib.abort
+    libc.src.stdlib.at_quick_exit
     libc.src.stdlib.atexit
     libc.src.stdlib.exit
     libc.src.stdlib.getenv
+    libc.src.stdlib.quick_exit
 
     # signal.h entrypoints
     libc.src.signal.raise
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index 71216530c4041b07f45fdbb2442dfcc52cd950f0..a4898724daf86da2def7952b0f491d79b4ba262f 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -180,6 +180,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fminimum_mag_num
     libc.src.math.fminimum_mag_numf
     libc.src.math.fminimum_mag_numl
+    libc.src.math.fmul
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst
index 5bbb056ec5c75a6c9573462a643a7a3110282da9..fec9b24bbd5815db3c8a39c399086c96a3292eb9 100644
--- a/libc/docs/c23.rst
+++ b/libc/docs/c23.rst
@@ -50,7 +50,7 @@ Additions:
   * issignaling
   * issubnormal
   * iszero
-  * llogb*
+  * llogb* |check|
   * pown*
   * powr*
   * rootn*
@@ -59,15 +59,17 @@ Additions:
   * ufromfp* |check|
   * fromfpx* |check|
   * ufromfpx* |check|
-  * nextup*
-  * nextdown*
+  * nextup* |check|
+  * nextdown* |check|
   * canonicalize* |check|
-  * fmaximum*
-  * fminimum*
-  * fmaximum_mag*
-  * fminimum_mag*
-  * fmaximum_mag_num*
-  * fminimum_mag_num*
+  * fmaximum* |check|
+  * fminimum* |check|
+  * fmaximum_mag* |check|
+  * fminimum_mag* |check|
+  * fmaximum_num* |check|
+  * fminimum_num* |check|
+  * fmaximum_mag_num* |check|
+  * fminimum_mag_num* |check|
   * fadd*
   * fsub*
   * fmul*
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index fd753742b522de0880b7d6883e5a1a5f8eff181f..3e122fb8bc26e40f82fb6b12af461d330229479f 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -136,31 +136,31 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | floor            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.2               | F.10.6.2                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmax             | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.2              | F.10.9.2                   |
+| fmax             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.2              | F.10.9.2                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum         | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.4              | F.10.9.4                   |
+| fmaximum         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.4              | F.10.9.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_mag     | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.6              | F.10.9.4                   |
+| fmaximum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.6              | F.10.9.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_mag_num | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.10             | F.10.9.5                   |
+| fmaximum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.10             | F.10.9.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_num     | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.8              | F.10.9.5                   |
+| fmaximum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.8              | F.10.9.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmin             | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.3              | F.10.9.3                   |
+| fmin             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.3              | F.10.9.3                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum         | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.5              | F.10.9.4                   |
+| fminimum         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.5              | F.10.9.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_mag     | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.7              | F.10.9.4                   |
+| fminimum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.7              | F.10.9.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_mag_num | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.11             | F.10.9.5                   |
+| fminimum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.11             | F.10.9.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_num     | |check|          | |check|         | |check|                |                      | |check|                | 7.12.12.9              | F.10.9.5                   |
+| fminimum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.9              | F.10.9.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmod             | |check|          | |check|         | |check|                |                      | |check|                | 7.12.10.1              | F.10.7.1                   |
+| fmod             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.10.1              | F.10.7.1                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmul             | N/A              |                 |                        | N/A                  |                        | 7.12.14.3              | F.10.11                    |
+| fmul             | N/A              | |check|         |                        | N/A                  |                        | 7.12.14.3              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| frexp            | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.7               | F.10.3.7                   |
+| frexp            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.7               | F.10.3.7                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fromfp           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.10              | F.10.6.10                  |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
@@ -168,39 +168,39 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | fsub             | N/A              |                 |                        | N/A                  |                        | 7.12.14.2              | F.10.11                    |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ilogb            | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.8               | F.10.3.8                   |
+| ilogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.8               | F.10.3.8                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | ldexp            | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.9               | F.10.3.9                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| llogb            | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.10              | F.10.3.10                  |
+| llogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.10              | F.10.3.10                  |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | llrint           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.5               | F.10.6.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | llround          | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.7               | F.10.6.7                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| logb             | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.17              | F.10.3.17                  |
+| logb             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.17              | F.10.3.17                  |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | lrint            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.5               | F.10.6.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | lround           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.7               | F.10.6.7                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| modf             | |check|          | |check|         | |check|                |                      | |check|                | 7.12.6.18              | F.10.3.18                  |
+| modf             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.18              | F.10.3.18                  |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nan              | |check|          | |check|         | |check|                |                      | |check|                | 7.12.11.2              | F.10.8.2                   |
+| nan              | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.2              | F.10.8.2                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | nearbyint        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.3               | F.10.6.3                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextafter        | |check|          | |check|         | |check|                |                      | |check|                | 7.12.11.3              | F.10.8.3                   |
+| nextafter        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.3              | F.10.8.3                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextdown         | |check|          | |check|         | |check|                |                      | |check|                | 7.12.11.6              | F.10.8.6                   |
+| nextdown         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.6              | F.10.8.6                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nexttoward       | |check|          | |check|         | |check|                |                      | N/A                    | 7.12.11.4              | F.10.8.4                   |
+| nexttoward       | |check|          | |check|         | |check|                | |check|              | N/A                    | 7.12.11.4              | F.10.8.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextup           | |check|          | |check|         | |check|                |                      | |check|                | 7.12.11.5              | F.10.8.5                   |
+| nextup           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.5              | F.10.8.5                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | remainder        | |check|          | |check|         | |check|                |                      |                        | 7.12.10.2              | F.10.7.2                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| remquo           | |check|          | |check|         | |check|                |                      |                        | 7.12.10.3              | F.10.7.3                   |
+| remquo           | |check|          | |check|         | |check|                |                      | |check|                | 7.12.10.3              | F.10.7.3                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | rint             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.4               | F.10.6.4                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 8e87642f66088b0c0062903aeb5d594aff9aecd4..9b3373a0ca39059650710209c2293ae745b3760d 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -117,3 +117,12 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.pid_t
     libc.include.sys_types
 )
+
+add_proxy_header_library(
+  atexithandler_t
+  HDRS
+    atexithandler_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.atexithandler_t
+    libc.include.stdlib
+)
diff --git a/libc/hdr/types/atexithandler_t.h b/libc/hdr/types/atexithandler_t.h
new file mode 100644
index 0000000000000000000000000000000000000000..4275e4407367433d86db4d1438390275fd3d34c5
--- /dev/null
+++ b/libc/hdr/types/atexithandler_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from atexithandler_t.h -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_ATEXITHANDLER_T_H
+#define LLVM_LIBC_HDR_ATEXITHANDLER_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/__atexithandler_t.h"
+
+#else // overlay mode
+
+#error // type not available in overlay mode
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_ATEXITHANDLER_T_H
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index ee2c910b85b000c3a3332da8c1b380d78cf07a87..356ac037770d25e47e4f09eecbf0435646dae278 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -58,7 +58,7 @@ add_header(pthread_rwlockattr_t HDR pthread_rwlockattr_t.h)
 add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type)
 add_header(rlim_t HDR rlim_t.h)
 add_header(time_t HDR time_t.h)
-add_header(stack_t HDR stack_t.h)
+add_header(stack_t HDR stack_t.h DEPENDS .size_t)
 add_header(suseconds_t HDR suseconds_t.h)
 add_header(struct_flock HDR struct_flock.h DEPENDS .off_t .pid_t)
 add_header(struct_flock64 HDR struct_flock64.h DEPENDS .off64_t .pid_t)
diff --git a/libc/spec/spec.td b/libc/spec/spec.td
index 056a3143c5a714e4d3cad84289439636dfb937df..7e1283e67fab2ff4761ffc0ea502db4ffff73a77 100644
--- a/libc/spec/spec.td
+++ b/libc/spec/spec.td
@@ -115,6 +115,7 @@ def IntPtr : PtrType<IntType>;
 def RestrictedIntPtr : RestrictedPtrType<IntType>;
 def FloatPtr : PtrType<FloatType>;
 def DoublePtr : PtrType<DoubleType>;
+def Float16Ptr : PtrType<Float16Type>;
 def Float128Ptr : PtrType<Float128Type>;
 def UnsignedCharPtr : PtrType<UnsignedCharType>;
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 0aadeb1a43a00b620f2b628189d6b9d9eed491f2..b5b6dbc481bd7e85e9e58fc1881ec82672e67ee2 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -416,63 +416,78 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"fminf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fminl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"fminf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+          GuardedFunctionSpec<"fminf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
 
           FunctionSpec<"fmax", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"fmaxf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+          GuardedFunctionSpec<"fmaxf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
 	  
 	  FunctionSpec<"fmaximum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaximumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaximuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximumf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fmaximumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fmaximum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaximum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fmaximum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fmaximum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaximum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaximum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_magf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fmaximum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fmaximum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaximum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaximum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_mag_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fmaximum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fminimum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fminimumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fminimuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimumf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fminimumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fminimum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fminimum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fminimum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fminimum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fminimum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fminimum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_magf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fminimum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 	  
 	  FunctionSpec<"fminimum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fminimum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fminimum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_mag_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fminimum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
+          FunctionSpec<"fmul", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+
+
           FunctionSpec<"fma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>]>,
 
           FunctionSpec<"fmod", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmodf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmodl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmodf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"fmodf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"frexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
+          GuardedFunctionSpec<"frexpf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"frexpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"fromfp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
@@ -505,11 +520,13 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"ilogb", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"ilogbf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"ilogbl", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"ilogbf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"ilogbf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"llogb", RetValSpec<LongType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"llogbf", RetValSpec<LongType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"llogbl", RetValSpec<LongType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"llogbf16", RetValSpec<LongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"llogbf128", RetValSpec<LongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"ldexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>]>,
@@ -532,11 +549,13 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"logb", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"logbf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"logbl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"logbf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"logbf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"modf", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoublePtr>]>,
           FunctionSpec<"modff", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatPtr>]>,
           FunctionSpec<"modfl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoublePtr>]>,
+          GuardedFunctionSpec<"modff16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"modff128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Ptr>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"cos", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
@@ -567,6 +586,7 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"remainderl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"remquof", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
+          GuardedFunctionSpec<"remquof128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT128">,
           FunctionSpec<"remquo", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"remquol", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
 
@@ -632,20 +652,24 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"nextafterf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"nextafter", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"nextafterl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"nextafterf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"nextafterf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"nexttowardf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<LongDoubleType>]>,
           FunctionSpec<"nexttoward", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<LongDoubleType>]>,
           FunctionSpec<"nexttowardl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"nexttowardf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
 
           FunctionSpec<"nextdown", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"nextdownf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"nextdownl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"nextdownf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"nextdownf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"nextup", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"nextupf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"nextupl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"nextupf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"nextupf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"powf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
@@ -676,6 +700,7 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"nanf", RetValSpec<FloatType>, [ArgSpec<ConstCharPtr>]>,
           FunctionSpec<"nan", RetValSpec<DoubleType>, [ArgSpec<ConstCharPtr>]>,
           FunctionSpec<"nanl", RetValSpec<LongDoubleType>, [ArgSpec<ConstCharPtr>]>,
+          GuardedFunctionSpec<"nanf16", RetValSpec<Float16Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"nanf128", RetValSpec<Float128Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"canonicalize", RetValSpec<IntType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
@@ -1093,8 +1118,9 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"free", RetValSpec<VoidType>, [ArgSpec<VoidPtr>]>,
 
           FunctionSpec<"_Exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
-          FunctionSpec<"exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
+          FunctionSpec<"at_quick_exit", RetValSpec<IntType>, [ArgSpec<AtexitHandlerT>]>,
           FunctionSpec<"atexit", RetValSpec<IntType>, [ArgSpec<AtexitHandlerT>]>,
+          FunctionSpec<"exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
           FunctionSpec<"quick_exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
       ]
   >;
diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt
index 08661aba5b6b188c1f169d40b708eb7f8a38d43d..e6f58b7cd82242eb8e22d971bff46222a6205f01 100644
--- a/libc/src/__support/CPP/CMakeLists.txt
+++ b/libc/src/__support/CPP/CMakeLists.txt
@@ -111,6 +111,7 @@ add_header_library(
     type_traits/add_lvalue_reference.h
     type_traits/add_pointer.h
     type_traits/add_rvalue_reference.h
+    type_traits/aligned_storage.h
     type_traits/always_false.h
     type_traits/bool_constant.h
     type_traits/conditional.h
diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h
index 1494aeb905e0935af897236b04431352ed3db704..d50b6612656dbb0e6a7013dc39fa343f074c8a5e 100644
--- a/libc/src/__support/CPP/type_traits.h
+++ b/libc/src/__support/CPP/type_traits.h
@@ -12,6 +12,7 @@
 #include "src/__support/CPP/type_traits/add_lvalue_reference.h"
 #include "src/__support/CPP/type_traits/add_pointer.h"
 #include "src/__support/CPP/type_traits/add_rvalue_reference.h"
+#include "src/__support/CPP/type_traits/aligned_storage.h"
 #include "src/__support/CPP/type_traits/bool_constant.h"
 #include "src/__support/CPP/type_traits/conditional.h"
 #include "src/__support/CPP/type_traits/decay.h"
diff --git a/libc/src/__support/CPP/type_traits/aligned_storage.h b/libc/src/__support/CPP/type_traits/aligned_storage.h
new file mode 100644
index 0000000000000000000000000000000000000000..574b1146f6b2ad38e03309aac6a988f7a8bf8ae0
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/aligned_storage.h
@@ -0,0 +1,27 @@
+//===-- aligned_storage type_traits    --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H
+
+#include <stddef.h> // size_t
+
+namespace LIBC_NAMESPACE::cpp {
+
+template <size_t Len, size_t Align> struct aligned_storage {
+  struct type {
+    alignas(Align) unsigned char data[Len];
+  };
+};
+
+template <size_t Len, size_t Align>
+using aligned_storage_t = typename aligned_storage<Len, Align>::type;
+
+} // namespace LIBC_NAMESPACE::cpp
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 01ca4254c79962d8c4d9ef7003d2fd46a418946d..1744f8cf626f314e7f7868b69209dd5b91ab1e2b 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -217,6 +217,7 @@ add_header_library(
     .nearest_integer_operations
     .normal_float
     libc.hdr.math_macros
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.limits
     libc.src.__support.CPP.type_traits
diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h
index d3c96d2d613d6f9286a9f0af1352878930bdc635..559ecde767c3031e793c5c6043d84ddaa676223a 100644
--- a/libc/src/__support/FPUtil/FPBits.h
+++ b/libc/src/__support/FPUtil/FPBits.h
@@ -744,7 +744,7 @@ public:
     if (LIBC_LIKELY(ep >= 0)) {
       // Implicit number bit will be removed by mask
       result.set_significand(number);
-      result.set_biased_exponent(ep + 1);
+      result.set_biased_exponent(static_cast<StorageType>(ep + 1));
     } else {
       result.set_significand(number >> -ep);
     }
diff --git a/libc/src/__support/FPUtil/ManipulationFunctions.h b/libc/src/__support/FPUtil/ManipulationFunctions.h
index a289c2ef70467bf16c1aaf952f72f17a93c49233..f695b83fb0b0ba936953c2a44dc4adbc8d3bbaeb 100644
--- a/libc/src/__support/FPUtil/ManipulationFunctions.h
+++ b/libc/src/__support/FPUtil/ManipulationFunctions.h
@@ -16,6 +16,7 @@
 #include "rounding_mode.h"
 
 #include "hdr/math_macros.h"
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/CPP/bit.h"
 #include "src/__support/CPP/limits.h" // INT_MAX, INT_MIN
 #include "src/__support/CPP/type_traits.h"
@@ -102,7 +103,7 @@ intlogb(U x) {
     return IntLogbConstants<T>::T_MAX;
   }
 
-  DyadicFloat<FPBits<U>::STORAGE_LEN> normal(bits.get_val());
+  DyadicFloat<cpp::max(FPBits<U>::STORAGE_LEN, 32)> normal(bits.get_val());
   int exponent = normal.get_unbiased_exponent();
   // The C standard does not specify the return value when an exponent is
   // out of int range. However, XSI conformance required that INT_MAX or
@@ -138,7 +139,7 @@ LIBC_INLINE constexpr T logb(T x) {
     return FPBits<T>::inf().get_val();
   }
 
-  DyadicFloat<FPBits<T>::STORAGE_LEN> normal(bits.get_val());
+  DyadicFloat<cpp::max(FPBits<T>::STORAGE_LEN, 32)> normal(bits.get_val());
   return static_cast<T>(normal.get_unbiased_exponent());
 }
 
diff --git a/libc/src/__support/FPUtil/NormalFloat.h b/libc/src/__support/FPUtil/NormalFloat.h
index 8bc1fecd653bfd48dabf4548dfda5a2c91b5d97c..33529d5e9b80a6bc02b383d123db1ec051dc7545 100644
--- a/libc/src/__support/FPUtil/NormalFloat.h
+++ b/libc/src/__support/FPUtil/NormalFloat.h
@@ -110,9 +110,11 @@ template <typename T> struct NormalFloat {
       if (shift <= FPBits<T>::FRACTION_LEN + 1) {
         // Generate a subnormal number. Might lead to loss of precision.
         // We round to nearest and round halfway cases to even.
-        const StorageType shift_out_mask = (StorageType(1) << shift) - 1;
+        const StorageType shift_out_mask =
+            static_cast<StorageType>(StorageType(1) << shift) - 1;
         const StorageType shift_out_value = mantissa & shift_out_mask;
-        const StorageType halfway_value = StorageType(1) << (shift - 1);
+        const StorageType halfway_value =
+            static_cast<StorageType>(StorageType(1) << (shift - 1));
         result.set_biased_exponent(0);
         result.set_mantissa(mantissa >> shift);
         StorageType new_mantissa = result.get_mantissa();
@@ -135,7 +137,8 @@ template <typename T> struct NormalFloat {
       }
     }
 
-    result.set_biased_exponent(exponent + FPBits<T>::EXP_BIAS);
+    result.set_biased_exponent(
+        static_cast<StorageType>(exponent + FPBits<T>::EXP_BIAS));
     result.set_mantissa(mantissa);
     return result.get_val();
   }
@@ -155,7 +158,7 @@ private:
     // Normalize subnormal numbers.
     if (bits.is_subnormal()) {
       unsigned shift = evaluate_normalization_shift(bits.get_mantissa());
-      mantissa = StorageType(bits.get_mantissa()) << shift;
+      mantissa = static_cast<StorageType>(bits.get_mantissa() << shift);
       exponent = 1 - FPBits<T>::EXP_BIAS - shift;
     } else {
       exponent = bits.get_biased_exponent() - FPBits<T>::EXP_BIAS;
diff --git a/libc/src/__support/FPUtil/generic/FMod.h b/libc/src/__support/FPUtil/generic/FMod.h
index 211ab926d28b0e1545f9fdb2b3d7591a65cb5014..f840a92b1a5a2c833bcf1282bc73125eb0fb8c64 100644
--- a/libc/src/__support/FPUtil/generic/FMod.h
+++ b/libc/src/__support/FPUtil/generic/FMod.h
@@ -210,7 +210,9 @@ private:
                     e_x - e_y <= int(FPB::EXP_LEN))) {
       StorageType m_x = sx.get_explicit_mantissa();
       StorageType m_y = sy.get_explicit_mantissa();
-      StorageType d = (e_x == e_y) ? (m_x - m_y) : (m_x << (e_x - e_y)) % m_y;
+      StorageType d = (e_x == e_y)
+                          ? (m_x - m_y)
+                          : static_cast<StorageType>(m_x << (e_x - e_y)) % m_y;
       if (d == 0)
         return FPB::zero();
       // iy - 1 because of "zero power" for number with power 1
diff --git a/libc/src/__support/fixedvector.h b/libc/src/__support/fixedvector.h
index 6aeb4d56363e939f483fa5d5dcf3f2f4344f442f..ddd0993a952720ec59899dd6c67a156a06aa39da 100644
--- a/libc/src/__support/fixedvector.h
+++ b/libc/src/__support/fixedvector.h
@@ -24,6 +24,17 @@ template <typename T, size_t CAPACITY> class FixedVector {
 public:
   constexpr FixedVector() = default;
 
+  using iterator = typename cpp::array<T, CAPACITY>::iterator;
+  constexpr FixedVector(iterator begin, iterator end) {
+    for (; begin != end; ++begin)
+      push_back(*begin);
+  }
+
+  constexpr FixedVector(size_t count, const T &value) {
+    for (size_t i = 0; i < count; ++i)
+      push_back(value);
+  }
+
   bool push_back(const T &obj) {
     if (item_count == CAPACITY)
       return false;
@@ -43,8 +54,14 @@ public:
     return true;
   }
 
+  T &operator[](size_t idx) { return store[idx]; }
+
+  const T &operator[](size_t idx) const { return store[idx]; }
+
   bool empty() const { return item_count == 0; }
 
+  size_t size() const { return item_count; }
+
   // Empties the store for all practical purposes.
   void reset() { item_count = 0; }
 
@@ -64,7 +81,6 @@ public:
   }
   LIBC_INLINE constexpr reverse_iterator rend() { return store.rend(); }
 
-  using iterator = typename cpp::array<T, CAPACITY>::iterator;
   LIBC_INLINE constexpr iterator begin() { return store.begin(); }
   LIBC_INLINE constexpr iterator end() { return iterator{&store[item_count]}; }
 };
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 5ae03b1f46c3245e55bf70592bca53b32f823505..f8582d8d42683310fca861de46e17daac2d54575 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -124,60 +124,74 @@ add_math_entrypoint_object(fmax)
 add_math_entrypoint_object(fmaxf)
 add_math_entrypoint_object(fmaxl)
 add_math_entrypoint_object(fmaxf128)
+add_math_entrypoint_object(fmaxf16)
 
 add_math_entrypoint_object(fmin)
 add_math_entrypoint_object(fminf)
 add_math_entrypoint_object(fminl)
 add_math_entrypoint_object(fminf128)
+add_math_entrypoint_object(fminf16)
 
 add_math_entrypoint_object(fmaximum)
 add_math_entrypoint_object(fmaximumf)
 add_math_entrypoint_object(fmaximuml)
+add_math_entrypoint_object(fmaximumf16)
 add_math_entrypoint_object(fmaximumf128)
 
 add_math_entrypoint_object(fmaximum_num)
 add_math_entrypoint_object(fmaximum_numf)
 add_math_entrypoint_object(fmaximum_numl)
+add_math_entrypoint_object(fmaximum_numf16)
 add_math_entrypoint_object(fmaximum_numf128)
 
 add_math_entrypoint_object(fmaximum_mag)
 add_math_entrypoint_object(fmaximum_magf)
 add_math_entrypoint_object(fmaximum_magl)
+add_math_entrypoint_object(fmaximum_magf16)
 add_math_entrypoint_object(fmaximum_magf128)
 
 add_math_entrypoint_object(fmaximum_mag_num)
 add_math_entrypoint_object(fmaximum_mag_numf)
 add_math_entrypoint_object(fmaximum_mag_numl)
+add_math_entrypoint_object(fmaximum_mag_numf16)
 add_math_entrypoint_object(fmaximum_mag_numf128)
 
 add_math_entrypoint_object(fminimum)
 add_math_entrypoint_object(fminimumf)
 add_math_entrypoint_object(fminimuml)
+add_math_entrypoint_object(fminimumf16)
 add_math_entrypoint_object(fminimumf128)
 
 add_math_entrypoint_object(fminimum_num)
 add_math_entrypoint_object(fminimum_numf)
 add_math_entrypoint_object(fminimum_numl)
+add_math_entrypoint_object(fminimum_numf16)
 add_math_entrypoint_object(fminimum_numf128)
 
 add_math_entrypoint_object(fminimum_mag)
 add_math_entrypoint_object(fminimum_magf)
 add_math_entrypoint_object(fminimum_magl)
+add_math_entrypoint_object(fminimum_magf16)
 add_math_entrypoint_object(fminimum_magf128)
 
 add_math_entrypoint_object(fminimum_mag_num)
 add_math_entrypoint_object(fminimum_mag_numf)
 add_math_entrypoint_object(fminimum_mag_numl)
+add_math_entrypoint_object(fminimum_mag_numf16)
 add_math_entrypoint_object(fminimum_mag_numf128)
 
+add_math_entrypoint_object(fmul)
+
 add_math_entrypoint_object(fmod)
 add_math_entrypoint_object(fmodf)
 add_math_entrypoint_object(fmodl)
+add_math_entrypoint_object(fmodf16)
 add_math_entrypoint_object(fmodf128)
 
 add_math_entrypoint_object(frexp)
 add_math_entrypoint_object(frexpf)
 add_math_entrypoint_object(frexpl)
+add_math_entrypoint_object(frexpf16)
 add_math_entrypoint_object(frexpf128)
 
 add_math_entrypoint_object(fromfp)
@@ -198,11 +212,13 @@ add_math_entrypoint_object(hypotf)
 add_math_entrypoint_object(ilogb)
 add_math_entrypoint_object(ilogbf)
 add_math_entrypoint_object(ilogbl)
+add_math_entrypoint_object(ilogbf16)
 add_math_entrypoint_object(ilogbf128)
 
 add_math_entrypoint_object(llogb)
 add_math_entrypoint_object(llogbf)
 add_math_entrypoint_object(llogbl)
+add_math_entrypoint_object(llogbf16)
 add_math_entrypoint_object(llogbf128)
 
 add_math_entrypoint_object(ldexp)
@@ -225,6 +241,7 @@ add_math_entrypoint_object(logf)
 add_math_entrypoint_object(logb)
 add_math_entrypoint_object(logbf)
 add_math_entrypoint_object(logbl)
+add_math_entrypoint_object(logbf16)
 add_math_entrypoint_object(logbf128)
 
 add_math_entrypoint_object(llrint)
@@ -254,11 +271,13 @@ add_math_entrypoint_object(lroundf128)
 add_math_entrypoint_object(modf)
 add_math_entrypoint_object(modff)
 add_math_entrypoint_object(modfl)
+add_math_entrypoint_object(modff16)
 add_math_entrypoint_object(modff128)
 
 add_math_entrypoint_object(nan)
 add_math_entrypoint_object(nanf)
 add_math_entrypoint_object(nanl)
+add_math_entrypoint_object(nanf16)
 add_math_entrypoint_object(nanf128)
 
 add_math_entrypoint_object(nearbyint)
@@ -270,20 +289,24 @@ add_math_entrypoint_object(nearbyintf128)
 add_math_entrypoint_object(nextafter)
 add_math_entrypoint_object(nextafterf)
 add_math_entrypoint_object(nextafterl)
+add_math_entrypoint_object(nextafterf16)
 add_math_entrypoint_object(nextafterf128)
 
 add_math_entrypoint_object(nexttoward)
 add_math_entrypoint_object(nexttowardf)
 add_math_entrypoint_object(nexttowardl)
+add_math_entrypoint_object(nexttowardf16)
 
 add_math_entrypoint_object(nextdown)
 add_math_entrypoint_object(nextdownf)
 add_math_entrypoint_object(nextdownl)
+add_math_entrypoint_object(nextdownf16)
 add_math_entrypoint_object(nextdownf128)
 
 add_math_entrypoint_object(nextup)
 add_math_entrypoint_object(nextupf)
 add_math_entrypoint_object(nextupl)
+add_math_entrypoint_object(nextupf16)
 add_math_entrypoint_object(nextupf128)
 
 add_math_entrypoint_object(pow)
@@ -295,6 +318,7 @@ add_math_entrypoint_object(remainderl)
 
 add_math_entrypoint_object(remquo)
 add_math_entrypoint_object(remquof)
+add_math_entrypoint_object(remquof128)
 add_math_entrypoint_object(remquol)
 
 add_math_entrypoint_object(rint)
diff --git a/libc/src/math/fmaxf16.h b/libc/src/math/fmaxf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..bf608f82c420ae03d5741bc4179292accb7fc7cf
--- /dev/null
+++ b/libc/src/math/fmaxf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmaxf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmaxf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXF16_H
diff --git a/libc/src/math/fmaximum_mag_numf16.h b/libc/src/math/fmaximum_mag_numf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..4c963d4dccc738689606ab5fb9c61cd23d741f24
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_numf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmaximum_mag_numf16 -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmaximum_mag_numf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H
diff --git a/libc/src/math/fmaximum_magf16.h b/libc/src/math/fmaximum_magf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..e5f57d3b7f1df4b966627564f5096e35896d39f7
--- /dev/null
+++ b/libc/src/math/fmaximum_magf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmaximum_magf16 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmaximum_magf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H
diff --git a/libc/src/math/fmaximum_numf16.h b/libc/src/math/fmaximum_numf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..b450a4595648315300afaec845e983fa2dfd900c
--- /dev/null
+++ b/libc/src/math/fmaximum_numf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmaximum_numf16 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmaximum_numf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H
diff --git a/libc/src/math/fmaximumf16.h b/libc/src/math/fmaximumf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..806339fde68309106f0886f55508999c1ab15751
--- /dev/null
+++ b/libc/src/math/fmaximumf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmaximumf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmaximumf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H
diff --git a/libc/src/math/fminf16.h b/libc/src/math/fminf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..22d4e6c8039488127f4b47a0e8392969d68ee807
--- /dev/null
+++ b/libc/src/math/fminf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fminf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINF16_H
+#define LLVM_LIBC_SRC_MATH_FMINF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fminf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINF16_H
diff --git a/libc/src/math/fminimum_mag_numf16.h b/libc/src/math/fminimum_mag_numf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..0fd314b2f5a222720e1a41c68b4d1fb8714371fb
--- /dev/null
+++ b/libc/src/math/fminimum_mag_numf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fminimum_mag_numf16 -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fminimum_mag_numf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H
diff --git a/libc/src/math/fminimum_magf16.h b/libc/src/math/fminimum_magf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..27673555403c9373c04b945717879a08327dd3c8
--- /dev/null
+++ b/libc/src/math/fminimum_magf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fminimum_magf16 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fminimum_magf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H
diff --git a/libc/src/math/fminimum_numf16.h b/libc/src/math/fminimum_numf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..598ff9d3c32d236c2ad63a7d181970c7dde3b3b7
--- /dev/null
+++ b/libc/src/math/fminimum_numf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fminimum_numf16 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fminimum_numf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H
diff --git a/libc/src/math/fminimumf16.h b/libc/src/math/fminimumf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..86dd240ae406d20ceb9aa36e717b4dd351b3dc72
--- /dev/null
+++ b/libc/src/math/fminimumf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fminimumf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUMF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUMF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fminimumf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUMF16_H
diff --git a/libc/src/math/fmodf16.h b/libc/src/math/fmodf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..ab658430275d845006559f9212e70d16db1c188d
--- /dev/null
+++ b/libc/src/math/fmodf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fmodf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMODF16_H
+#define LLVM_LIBC_SRC_MATH_FMODF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 fmodf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMODF16_H
diff --git a/libc/src/math/fmul.h b/libc/src/math/fmul.h
new file mode 100644
index 0000000000000000000000000000000000000000..fbc1069db733e02f7995776d47077311cae41b02
--- /dev/null
+++ b/libc/src/math/fmul.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmul --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMUL_H
+#define LLVM_LIBC_SRC_MATH_FMUL_H
+
+namespace LIBC_NAMESPACE {
+
+float fmul(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMUL_H
diff --git a/libc/src/math/frexpf16.h b/libc/src/math/frexpf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..dc1898cc5f702d6cf206f78c44361eca0179b225
--- /dev/null
+++ b/libc/src/math/frexpf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for frexpf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FREXPF16_H
+#define LLVM_LIBC_SRC_MATH_FREXPF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 frexpf16(float16 x, int *exp);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FREXPF16_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 95904bef93d24b9a4c4f019719558ef7689e5c67..caaa0ac23dc7a08820205e85522100c8e9746e80 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1301,6 +1301,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  frexpf16
+  SRCS
+    frexpf16.cpp
+  HDRS
+    ../frexpf16.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_entrypoint_object(
   frexpf128
   SRCS
@@ -1350,6 +1363,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  ilogbf16
+  SRCS
+    ilogbf16.cpp
+  HDRS
+    ../ilogbf16.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_entrypoint_object(
   ilogbf128
   SRCS
@@ -1399,6 +1425,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  llogbf16
+  SRCS
+    llogbf16.cpp
+  HDRS
+    ../llogbf16.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_entrypoint_object(
   llogbf128
   SRCS
@@ -1672,6 +1711,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  logbf16
+  SRCS
+    logbf16.cpp
+  HDRS
+    ../logbf16.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_entrypoint_object(
   logbf128
   SRCS
@@ -1681,6 +1733,7 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     -O3
   DEPENDS
+    libc.src.__support.macros.properties.types
     libc.src.__support.FPUtil.manipulation_functions
 )
 
@@ -1720,6 +1773,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  modff16
+  SRCS
+    modff16.cpp
+  HDRS
+    ../modff16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   modff128
   SRCS
@@ -1782,6 +1848,20 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fminf16
+  SRCS
+    fminf16.cpp
+  HDRS
+    ../fminf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+
 add_entrypoint_object(
   fmax
   SRCS
@@ -1831,6 +1911,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fmaxf16
+  SRCS
+    fmaxf16.cpp
+  HDRS
+    ../fmaxf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmaximum
   SRCS
@@ -1867,6 +1960,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fmaximumf16
+  SRCS
+    fmaximumf16.cpp
+  HDRS
+    ../fmaximumf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmaximumf128
   SRCS
@@ -1916,6 +2022,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fmaximum_numf16
+  SRCS
+    fmaximum_numf16.cpp
+  HDRS
+    ../fmaximum_numf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmaximum_numf128
   SRCS
@@ -1965,6 +2084,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fmaximum_magf16
+  SRCS
+    fmaximum_magf16.cpp
+  HDRS
+    ../fmaximum_magf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmaximum_magf128
   SRCS
@@ -1978,7 +2110,6 @@ add_entrypoint_object(
     -O3
 )
 
-
 add_entrypoint_object(
   fmaximum_mag_num
   SRCS
@@ -2015,6 +2146,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fmaximum_mag_numf16
+  SRCS
+    fmaximum_mag_numf16.cpp
+  HDRS
+    ../fmaximum_mag_numf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmaximum_mag_numf128
   SRCS
@@ -2064,6 +2208,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fminimumf16
+  SRCS
+    fminimumf16.cpp
+  HDRS
+    ../fminimumf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fminimumf128
   SRCS
@@ -2113,6 +2270,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fminimum_numf16
+  SRCS
+    fminimum_numf16.cpp
+  HDRS
+    ../fminimum_numf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fminimum_numf128
   SRCS
@@ -2162,6 +2332,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fminimum_magf16
+  SRCS
+    fminimum_magf16.cpp
+  HDRS
+    ../fminimum_magf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fminimum_magf128
   SRCS
@@ -2175,7 +2358,6 @@ add_entrypoint_object(
     -O3
 )
 
-
 add_entrypoint_object(
   fminimum_mag_num
   SRCS
@@ -2212,6 +2394,19 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  fminimum_mag_numf16
+  SRCS
+    fminimum_mag_numf16.cpp
+  HDRS
+    ../fminimum_mag_numf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fminimum_mag_numf128
   SRCS
@@ -2225,6 +2420,22 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fmul
+  SRCS
+    fmul.cpp
+  HDRS
+    ../fmul.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.uint128
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.rounding_mode
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   sqrt
   SRCS
@@ -2287,6 +2498,18 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  remquof128
+  SRCS
+    remquof128.cpp
+  HDRS
+    ../remquof128.h
+  DEPENDS
+    libc.src.__support.FPUtil.division_and_remainder_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   remquo
   SRCS
@@ -2474,6 +2697,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  nanf16
+  SRCS
+    nanf16.cpp
+  HDRS
+    ../nanf16.h
+  DEPENDS
+    libc.src.__support.str_to_float
+    libc.src.errno.errno
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   nanf128
   SRCS
@@ -2523,6 +2759,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  nextafterf16
+  SRCS
+    nextafterf16.cpp
+  HDRS
+    ../nextafterf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   nextafterf128
   SRCS
@@ -2572,6 +2821,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  nexttowardf16
+  SRCS
+    nexttowardf16.cpp
+  HDRS
+    ../nexttowardf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   nextdown
   SRCS
@@ -2608,6 +2870,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  nextdownf16
+  SRCS
+    nextdownf16.cpp
+  HDRS
+    ../nextdownf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   nextdownf128
   SRCS
@@ -2657,6 +2932,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  nextupf16
+  SRCS
+    nextupf16.cpp
+  HDRS
+    ../nextupf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.manipulation_functions
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   nextupf128
   SRCS
@@ -2706,6 +2994,19 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fmodf16
+  SRCS
+    fmodf16.cpp
+  HDRS
+    ../fmodf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.generic.fmod
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   fmodf128
   SRCS
diff --git a/libc/src/math/generic/fmaxf16.cpp b/libc/src/math/generic/fmaxf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c317aef3cb3b8b4c8afee22f6a6a2a72f9588fba
--- /dev/null
+++ b/libc/src/math/generic/fmaxf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaxf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaxf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmaxf16, (float16 x, float16 y)) {
+  return fputil::fmax(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag_numf16.cpp b/libc/src/math/generic/fmaximum_mag_numf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5055802c4cf85b0a9ed0d917b7aecfa7153f7c43
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_numf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_mag_numf16 function --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_numf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmaximum_mag_numf16, (float16 x, float16 y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_magf16.cpp b/libc/src/math/generic/fmaximum_magf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fbd5eaccf3093118351cdc03d12a90fef23c0358
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_magf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_magf16 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_magf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmaximum_magf16, (float16 x, float16 y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_numf16.cpp b/libc/src/math/generic/fmaximum_numf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..187cfbeee6e2d85d03efe71784860a214d0b90ed
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_numf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_numf16 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_numf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmaximum_numf16, (float16 x, float16 y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximumf16.cpp b/libc/src/math/generic/fmaximumf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9e194d2ecef6932962c76a66e00f69ccdfa0c348
--- /dev/null
+++ b/libc/src/math/generic/fmaximumf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximumf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximumf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmaximumf16, (float16 x, float16 y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminf16.cpp b/libc/src/math/generic/fminf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..12547c33c636166a90d56f6c27cd059e607a7c2e
--- /dev/null
+++ b/libc/src/math/generic/fminf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fminf16, (float16 x, float16 y)) {
+  return fputil::fmin(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag_numf16.cpp b/libc/src/math/generic/fminimum_mag_numf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1a893c6c4bbc183f14016d7bade23937702964f3
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_numf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_mag_numf16 function --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_numf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fminimum_mag_numf16, (float16 x, float16 y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_magf16.cpp b/libc/src/math/generic/fminimum_magf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..45183a963e2d8892f19d39c2aeff01c1fadbf599
--- /dev/null
+++ b/libc/src/math/generic/fminimum_magf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_magf16 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_magf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fminimum_magf16, (float16 x, float16 y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_numf16.cpp b/libc/src/math/generic/fminimum_numf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..825ad3e7b63af0468a1b21cef6add259b5d81fc3
--- /dev/null
+++ b/libc/src/math/generic/fminimum_numf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_numf16 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_numf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fminimum_numf16, (float16 x, float16 y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimumf16.cpp b/libc/src/math/generic/fminimumf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..16f738be7e58c5cb881a4f0620f0ef16d5497891
--- /dev/null
+++ b/libc/src/math/generic/fminimumf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimumf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimumf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fminimumf16, (float16 x, float16 y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmodf16.cpp b/libc/src/math/generic/fmodf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a5bfd78113f63be77034c424dfa870f8b006ceeb
--- /dev/null
+++ b/libc/src/math/generic/fmodf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmodf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmodf16.h"
+#include "src/__support/FPUtil/generic/FMod.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, fmodf16, (float16 x, float16 y)) {
+  return fputil::generic::FMod<float16, uint32_t>::eval(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmul.cpp b/libc/src/math/generic/fmul.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..40af20466a665d702f2395122fbc6203916dafeb
--- /dev/null
+++ b/libc/src/math/generic/fmul.cpp
@@ -0,0 +1,128 @@
+//===-- Implementation of fmul function------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmul.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/common.h"
+#include "src/__support/uint128.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fmul, (double x, double y)) {
+  auto x_bits = fputil::FPBits<double>(x);
+
+  auto y_bits = fputil::FPBits<double>(y);
+
+  auto output_sign = (x_bits.sign() != y_bits.sign()) ? Sign::NEG : Sign::POS;
+
+  if (LIBC_UNLIKELY(x_bits.is_inf_or_nan() || y_bits.is_inf_or_nan() ||
+                    x_bits.is_zero() || y_bits.is_zero())) {
+    if (x_bits.is_nan())
+      return static_cast<float>(x);
+    if (y_bits.is_nan())
+      return static_cast<float>(y);
+    if (x_bits.is_inf())
+      return y_bits.is_zero()
+                 ? fputil::FPBits<float>::quiet_nan().get_val()
+                 : fputil::FPBits<float>::inf(output_sign).get_val();
+    if (y_bits.is_inf())
+      return x_bits.is_zero()
+                 ? fputil::FPBits<float>::quiet_nan().get_val()
+                 : fputil::FPBits<float>::inf(output_sign).get_val();
+    // Now either x or y is zero, and the other one is finite.
+    return fputil::FPBits<float>::zero(output_sign).get_val();
+  }
+
+  uint64_t mx, my;
+
+  // Get mantissa and append the hidden bit if needed.
+  mx = x_bits.get_explicit_mantissa();
+  my = y_bits.get_explicit_mantissa();
+
+  // Get the corresponding biased exponent.
+  int ex = x_bits.get_explicit_exponent();
+  int ey = y_bits.get_explicit_exponent();
+
+  // Count the number of leading zeros of the explicit mantissas.
+  int nx = cpp::countl_zero(mx);
+  int ny = cpp::countl_zero(my);
+  // Shift the leading 1 bit to the most significant bit.
+  mx <<= nx;
+  my <<= ny;
+
+  // Adjust exponent accordingly: If x or y are normal, we will only need to
+  // shift by (exponent length + sign bit = 11 bits. If x or y are denormal, we
+  // will need to shift more than 11 bits.
+  ex -= (nx - 11);
+  ey -= (ny - 11);
+
+  UInt128 product = static_cast<UInt128>(mx) * static_cast<UInt128>(my);
+  int32_t dm1;
+  uint64_t highs, lows;
+  uint64_t g, hight, lowt;
+  uint32_t m;
+  uint32_t b;
+  int c;
+
+  highs = static_cast<uint64_t>(product >> 64);
+  c = static_cast<int>(highs >= 0x8000000000000000);
+  lows = static_cast<uint64_t>(product);
+
+  lowt = (lows != 0);
+
+  dm1 = ex + ey + c + fputil::FPBits<float>::EXP_BIAS;
+
+  int round_mode = fputil::quick_get_round();
+  if (dm1 >= 255) {
+    if ((round_mode == FE_TOWARDZERO) ||
+        (round_mode == FE_UPWARD && output_sign.is_neg()) ||
+        (round_mode == FE_DOWNWARD && output_sign.is_pos())) {
+      return fputil::FPBits<float>::max_normal(output_sign).get_val();
+    }
+    return fputil::FPBits<float>::inf().get_val();
+  } else if (dm1 <= 0) {
+
+    int m_shift = 40 + c - dm1;
+    int g_shift = m_shift - 1;
+    int h_shift = 64 - g_shift;
+    m = (m_shift >= 64) ? 0 : static_cast<uint32_t>(highs >> m_shift);
+
+    g = g_shift >= 64 ? 0 : (highs >> g_shift) & 1;
+    hight = h_shift >= 64 ? highs : (highs << h_shift) != 0;
+
+    dm1 = 0;
+  } else {
+    m = static_cast<uint32_t>(highs >> (39 + c));
+    g = (highs >> (38 + c)) & 1;
+    hight = (highs << (26 - c)) != 0;
+  }
+
+  if (round_mode == FE_TONEAREST) {
+    b = g && ((hight && lowt) || ((m & 1) != 0));
+  } else if ((output_sign.is_neg() && round_mode == FE_DOWNWARD) ||
+             (output_sign.is_pos() && round_mode == FE_UPWARD)) {
+    b = (g == 0 && (hight && lowt) == 0) ? 0 : 1;
+  } else {
+    b = 0;
+  }
+
+  uint32_t exp16 = (dm1 << 23);
+
+  uint32_t m2 = m & fputil::FPBits<float>::FRACTION_MASK;
+
+  uint32_t result = (exp16 + m2) + b;
+
+  auto result_bits = fputil::FPBits<float>(result);
+  result_bits.set_sign(output_sign);
+  return result_bits.get_val();
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/frexpf16.cpp b/libc/src/math/generic/frexpf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2d29c07c7217dd5aec08dc056f0b5d765a3896fa
--- /dev/null
+++ b/libc/src/math/generic/frexpf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of frexpf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/frexpf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, frexpf16, (float16 x, int *exp)) {
+  return fputil::frexp(x, *exp);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ilogbf16.cpp b/libc/src/math/generic/ilogbf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..87e43f8a5ada3a2a2ebe0d8f8c0cf209a18be5b2
--- /dev/null
+++ b/libc/src/math/generic/ilogbf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ilogbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ilogbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, ilogbf16, (float16 x)) {
+  return fputil::intlogb<int>(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/llogbf16.cpp b/libc/src/math/generic/llogbf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b7a21b95604d751888e1265fbaabe1509a16f11e
--- /dev/null
+++ b/libc/src/math/generic/llogbf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of llogbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llogbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, llogbf16, (float16 x)) {
+  return fputil::intlogb<long>(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/logbf16.cpp b/libc/src/math/generic/logbf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..52eb9ac6a342d10836b2b0ccd619c83217f12767
--- /dev/null
+++ b/libc/src/math/generic/logbf16.cpp
@@ -0,0 +1,17 @@
+//===-- Implementation of logbf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/logbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, logbf16, (float16 x)) { return fputil::logb(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/modff16.cpp b/libc/src/math/generic/modff16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..50cc5b5783adee122b56abadc132b863de58bfed
--- /dev/null
+++ b/libc/src/math/generic/modff16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of modff16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/modff16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, modff16, (float16 x, float16 *iptr)) {
+  return fputil::modf(x, *iptr);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/nanf16.cpp b/libc/src/math/generic/nanf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c42cd25fe3ef0c750decb82686913758eb18cb2b
--- /dev/null
+++ b/libc/src/math/generic/nanf16.cpp
@@ -0,0 +1,23 @@
+//===-- Implementation of nanf16 function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nanf16.h"
+#include "src/__support/common.h"
+#include "src/__support/str_to_float.h"
+#include "src/errno/libc_errno.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, nanf16, (const char *arg)) {
+  auto result = internal::strtonan<float16>(arg);
+  if (result.has_error())
+    libc_errno = result.error;
+  return result.value;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/nextafterf16.cpp b/libc/src/math/generic/nextafterf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..144b3fc6146148091ea7ecb8095e36e2f8e7aa3d
--- /dev/null
+++ b/libc/src/math/generic/nextafterf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of nextafterf16 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextafterf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, nextafterf16, (float16 x, float16 y)) {
+  return fputil::nextafter(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/nextdownf16.cpp b/libc/src/math/generic/nextdownf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9fdaa9dafdd8b2ad51c390f9df698a27e44ded2c
--- /dev/null
+++ b/libc/src/math/generic/nextdownf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of nextdownf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextdownf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, nextdownf16, (float16 x)) {
+  return fputil::nextupdown</*IsDown=*/true>(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/nexttowardf16.cpp b/libc/src/math/generic/nexttowardf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d1d78e8f22d3e6f4f0bd33af93513c6b04b2c201
--- /dev/null
+++ b/libc/src/math/generic/nexttowardf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of nexttowardf16 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nexttowardf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, nexttowardf16, (float16 x, long double y)) {
+  // We can reuse the nextafter implementation because the internal nextafter is
+  // templated on the types of the arguments.
+  return fputil::nextafter(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/nextupf16.cpp b/libc/src/math/generic/nextupf16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5d3d52c94068f099136f7558cdda3e1a42161b8d
--- /dev/null
+++ b/libc/src/math/generic/nextupf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of nextupf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextupf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float16, nextupf16, (float16 x)) {
+  return fputil::nextupdown</*IsDown=*/false>(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/remquof128.cpp b/libc/src/math/generic/remquof128.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e195c7b51b5ff22772527a9390c3c5929c92ef11
--- /dev/null
+++ b/libc/src/math/generic/remquof128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of remquof128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/remquof128.h"
+#include "src/__support/FPUtil/DivisionAndRemainderOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, remquof128, (float128 x, float128 y, int *exp)) {
+  return fputil::remquo(x, y, *exp);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/ilogbf16.h b/libc/src/math/ilogbf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..4884a140cb4f6434749e47f1320a898cebccc57b
--- /dev/null
+++ b/libc/src/math/ilogbf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ilogbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ILOGBF16_H
+#define LLVM_LIBC_SRC_MATH_ILOGBF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+int ilogbf16(float16 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ILOGBF16_H
diff --git a/libc/src/math/llogbf16.h b/libc/src/math/llogbf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..267ae410a31d8e199c3629a597b563f0b587bc2a
--- /dev/null
+++ b/libc/src/math/llogbf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for llogbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLOGBF16_H
+#define LLVM_LIBC_SRC_MATH_LLOGBF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long llogbf16(float16 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LLOGBF16_H
diff --git a/libc/src/math/logbf16.h b/libc/src/math/logbf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..8082e06c33d524256c562c09c7ef1c01fd376287
--- /dev/null
+++ b/libc/src/math/logbf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for logbf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LOGBF16_H
+#define LLVM_LIBC_SRC_MATH_LOGBF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 logbf16(float16 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LOGBF16_H
diff --git a/libc/src/math/modff16.h b/libc/src/math/modff16.h
new file mode 100644
index 0000000000000000000000000000000000000000..a3017c5a92a6427f7a3163f24d0ec34ed0669590
--- /dev/null
+++ b/libc/src/math/modff16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for modff16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_MODFF16_H
+#define LLVM_LIBC_SRC_MATH_MODFF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 modff16(float16 x, float16 *iptr);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_MODFF16_H
diff --git a/libc/src/math/nanf16.h b/libc/src/math/nanf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..c2db4bac0a15515eee107a4057d1f636b1b32ce4
--- /dev/null
+++ b/libc/src/math/nanf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for nanf16 ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NANF16_H
+#define LLVM_LIBC_SRC_MATH_NANF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 nanf16(const char *arg);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_NANF16_H
diff --git a/libc/src/math/nextafterf16.h b/libc/src/math/nextafterf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..293569ef40c53b998594ed2e952fdd0587657d0a
--- /dev/null
+++ b/libc/src/math/nextafterf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for nextafterf16 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 nextafterf16(float16 x, float16 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H
diff --git a/libc/src/math/nextdownf16.h b/libc/src/math/nextdownf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..19137574ac92f628d287a3d6263b746934622162
--- /dev/null
+++ b/libc/src/math/nextdownf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for nextdownf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 nextdownf16(float16 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H
diff --git a/libc/src/math/nexttowardf16.h b/libc/src/math/nexttowardf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..604eb32c25778cdc0cd3ab13f951dedd68baa049
--- /dev/null
+++ b/libc/src/math/nexttowardf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for nexttowardf16 -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 nexttowardf16(float16 x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H
diff --git a/libc/src/math/nextupf16.h b/libc/src/math/nextupf16.h
new file mode 100644
index 0000000000000000000000000000000000000000..b2973e4afc25615919d0d93cc59842b3cf1c8000
--- /dev/null
+++ b/libc/src/math/nextupf16.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for nextupf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTUPF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTUPF16_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float16 nextupf16(float16 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTUPF16_H
diff --git a/libc/src/math/remquof128.h b/libc/src/math/remquof128.h
new file mode 100644
index 0000000000000000000000000000000000000000..e9db1ef5c5b510a58071d6fd6d2158ba2ea8066c
--- /dev/null
+++ b/libc/src/math/remquof128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for remquof128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_REMQUOF128_H
+#define LLVM_LIBC_SRC_MATH_REMQUOF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 remquof128(float128 x, float128 y, int *exp);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_REMQUOF128_H
diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt
index 1056f38fc7513acdf3d15cc41d7c81d7040b3ee1..ee48e441d1c59f4a57ac246ed6fa6bd8fabe4251 100644
--- a/libc/src/stdio/CMakeLists.txt
+++ b/libc/src/stdio/CMakeLists.txt
@@ -159,29 +159,6 @@ add_entrypoint_object(
     libc.src.stdio.printf_core.writer
 )
 
-list(APPEND printf_deps
-      libc.src.__support.arg_list
-      libc.src.stdio.printf_core.vfprintf_internal
-)
-
-if(LLVM_LIBC_FULL_BUILD)
-  list(APPEND printf_deps
-      libc.src.__support.File.file
-      libc.src.__support.File.platform_file
-      libc.src.__support.File.platform_stdout
-  )
-endif()
-
-add_entrypoint_object(
-  printf
-  SRCS
-    printf.cpp
-  HDRS
-    printf.h
-  DEPENDS
-    ${printf_deps}
-)
-
 add_entrypoint_object(
   fprintf
   SRCS
@@ -215,16 +192,6 @@ add_entrypoint_object(
     libc.src.stdio.printf_core.writer
 )
 
-add_entrypoint_object(
-  vprintf
-  SRCS
-    vprintf.cpp
-  HDRS
-    vprintf.h
-  DEPENDS
-    ${printf_deps}
-)
-
 add_entrypoint_object(
   vfprintf
   SRCS
@@ -286,6 +253,7 @@ add_stdio_entrypoint_object(fwrite)
 add_stdio_entrypoint_object(fputc)
 add_stdio_entrypoint_object(putc)
 add_stdio_entrypoint_object(putchar)
+add_stdio_entrypoint_object(printf)
 add_stdio_entrypoint_object(fgetc)
 add_stdio_entrypoint_object(fgetc_unlocked)
 add_stdio_entrypoint_object(getc)
@@ -297,3 +265,4 @@ add_stdio_entrypoint_object(ungetc)
 add_stdio_entrypoint_object(stdin)
 add_stdio_entrypoint_object(stdout)
 add_stdio_entrypoint_object(stderr)
+add_stdio_entrypoint_object(vprintf)
diff --git a/libc/src/stdio/baremetal/CMakeLists.txt b/libc/src/stdio/baremetal/CMakeLists.txt
index 65089274050bddff0d75bff7e4ab6687d0c3a8e0..a1a5ba5faaf3daff075b543d3d4d5078e8e496c9 100644
--- a/libc/src/stdio/baremetal/CMakeLists.txt
+++ b/libc/src/stdio/baremetal/CMakeLists.txt
@@ -7,3 +7,15 @@ add_entrypoint_object(
   DEPENDS
     libc.include.stdio
 )
+
+add_entrypoint_object(
+  printf
+  SRCS
+    printf.cpp
+  HDRS
+    ../printf.h
+  DEPENDS
+    libc.src.stdio.printf_core.printf_main
+    libc.src.stdio.printf_core.writer
+    libc.src.__support.arg_list
+)
diff --git a/libc/src/stdio/baremetal/printf.cpp b/libc/src/stdio/baremetal/printf.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..597078b1ede513ace8f5aa3146aa2e7115b20c6a
--- /dev/null
+++ b/libc/src/stdio/baremetal/printf.cpp
@@ -0,0 +1,57 @@
+//===-- Implementation of printf for baremetal ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdio/printf.h"
+#include "src/__support/arg_list.h"
+#include "src/stdio/printf_core/core_structs.h"
+#include "src/stdio/printf_core/printf_main.h"
+#include "src/stdio/printf_core/writer.h"
+
+#include <stdarg.h>
+
+// TODO(https://github.com/llvm/llvm-project/issues/94685) unify baremetal hooks
+
+// This is intended to be provided by the vendor.
+extern "C" size_t __llvm_libc_raw_write(const char *s, size_t size);
+
+namespace LIBC_NAMESPACE {
+
+namespace {
+
+LIBC_INLINE int raw_write_hook(cpp::string_view new_str, void *) {
+  size_t written = __llvm_libc_raw_write(new_str.data(), new_str.size());
+  if (written != new_str.size())
+    return printf_core::FILE_WRITE_ERROR;
+  return printf_core::WRITE_OK;
+}
+
+} // namespace
+
+LLVM_LIBC_FUNCTION(int, printf, (const char *__restrict format, ...)) {
+  va_list vlist;
+  va_start(vlist, format);
+  internal::ArgList args(vlist); // This holder class allows for easier copying
+                                 // and pointer semantics, as well as handling
+                                 // destruction automatically.
+  va_end(vlist);
+  constexpr size_t BUFF_SIZE = 1024;
+  char buffer[BUFF_SIZE];
+
+  printf_core::WriteBuffer wb(buffer, BUFF_SIZE, &raw_write_hook, nullptr);
+  printf_core::Writer writer(&wb);
+
+  int retval = printf_core::printf_main(&writer, format, args);
+
+  int flushval = wb.overflow_write("");
+  if (flushval != printf_core::WRITE_OK)
+    retval = flushval;
+
+  return retval;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdio/generic/CMakeLists.txt b/libc/src/stdio/generic/CMakeLists.txt
index ae255917adfe3ac82d65878c5435ca350ed8f32c..9cd4cfdae17f4cbe3a1fad924ca120444b4773e6 100644
--- a/libc/src/stdio/generic/CMakeLists.txt
+++ b/libc/src/stdio/generic/CMakeLists.txt
@@ -363,6 +363,39 @@ add_entrypoint_object(
     libc.src.__support.File.platform_file
 )
 
+list(APPEND printf_deps
+      libc.src.__support.arg_list
+      libc.src.stdio.printf_core.vfprintf_internal
+)
+
+if(LLVM_LIBC_FULL_BUILD)
+  list(APPEND printf_deps
+      libc.src.__support.File.file
+      libc.src.__support.File.platform_file
+      libc.src.__support.File.platform_stdout
+  )
+endif()
+
+add_entrypoint_object(
+  printf
+  SRCS
+    printf.cpp
+  HDRS
+    ../printf.h
+  DEPENDS
+    ${printf_deps}
+)
+
+add_entrypoint_object(
+  vprintf
+  SRCS
+    vprintf.cpp
+  HDRS
+    ../vprintf.h
+  DEPENDS
+    ${printf_deps}
+)
+
 add_entrypoint_object(
   fgets
   SRCS
diff --git a/libc/src/stdio/printf.cpp b/libc/src/stdio/generic/printf.cpp
similarity index 100%
rename from libc/src/stdio/printf.cpp
rename to libc/src/stdio/generic/printf.cpp
diff --git a/libc/src/stdio/vprintf.cpp b/libc/src/stdio/generic/vprintf.cpp
similarity index 100%
rename from libc/src/stdio/vprintf.cpp
rename to libc/src/stdio/generic/vprintf.cpp
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index e0bff5198b590ce41d57e07975052a7429bcf99f..f0091ad367c0a4dc5b5f5dc49b26af8578063bf0 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -50,6 +50,7 @@ add_entrypoint_object(
     quick_exit.h
   DEPENDS
     libc.src.__support.OSUtil.osutil
+    .exit_handler
 )
 
 add_entrypoint_object(
@@ -415,14 +416,17 @@ add_entrypoint_object(
     libc.src.__support.OSUtil.osutil
 )
 
-add_entrypoint_object(
-  atexit
+# TODO: Move all exit functions to linux specific 
+
+if (TARGET libc.src.__support.threads.mutex)
+add_object_library(
+  exit_handler
   SRCS
-    atexit.cpp
+    exit_handler.cpp
   HDRS
-    atexit.h
+    exit_handler.h
   CXX_STANDARD
-    20 # For constinit of the atexit callback list.
+    20 # For constinit 
   DEPENDS
     libc.src.__support.CPP.mutex
     libc.src.__support.CPP.new
@@ -431,6 +435,27 @@ add_entrypoint_object(
     libc.src.__support.fixedvector
     libc.src.__support.threads.mutex
 )
+endif()
+
+add_entrypoint_object(
+  atexit
+  SRCS
+    atexit.cpp
+  HDRS
+    atexit.h
+  DEPENDS
+    .exit_handler
+)
+
+add_entrypoint_object(
+  at_quick_exit
+  SRCS
+    at_quick_exit.cpp
+  HDRS
+    at_quick_exit.h
+  DEPENDS
+    .exit_handler
+)
 
 add_entrypoint_object(
   exit
@@ -442,6 +467,7 @@ add_entrypoint_object(
     ._Exit
     .atexit
     libc.src.__support.OSUtil.osutil
+    .exit_handler
 )
 
 add_entrypoint_object(
diff --git a/libc/src/stdlib/at_quick_exit.cpp b/libc/src/stdlib/at_quick_exit.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..752d67e7fe44dc3162118b6e9555b1ee7081d0a9
--- /dev/null
+++ b/libc/src/stdlib/at_quick_exit.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of at_quick_exit -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/at_quick_exit.h"
+#include "hdr/types/atexithandler_t.h"
+#include "src/__support/common.h"
+#include "src/stdlib/exit_handler.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, at_quick_exit, (__atexithandler_t callback)) {
+  return add_atexit_unit(
+      at_quick_exit_callbacks,
+      {&stdc_at_exit_func, reinterpret_cast<void *>(callback)});
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdlib/at_quick_exit.h b/libc/src/stdlib/at_quick_exit.h
new file mode 100644
index 0000000000000000000000000000000000000000..c36c797088aba22e21060e1a61ea5b4bc416560e
--- /dev/null
+++ b/libc/src/stdlib/at_quick_exit.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for at_quick_exit -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H
+#define LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H
+
+#include "hdr/types/atexithandler_t.h"
+
+namespace LIBC_NAMESPACE {
+
+int at_quick_exit(__atexithandler_t);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H
diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp
index 9e37c4cf256c9ef15e207d7ce16eb1ff7509a209..ca3cbfe87a88c5ef9e696b023d881d639ca952b7 100644
--- a/libc/src/stdlib/atexit.cpp
+++ b/libc/src/stdlib/atexit.cpp
@@ -7,95 +7,28 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdlib/atexit.h"
-#include "src/__support/CPP/mutex.h" // lock_guard
-#include "src/__support/blockstore.h"
+#include "hdr/types/atexithandler_t.h"
 #include "src/__support/common.h"
-#include "src/__support/fixedvector.h"
-#include "src/__support/threads/mutex.h"
+#include "src/stdlib/exit_handler.h"
 
 namespace LIBC_NAMESPACE {
 
-namespace {
-
-Mutex handler_list_mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
-                       /*pshared=*/false);
-
-using AtExitCallback = void(void *);
-using StdCAtExitCallback = void(void);
-
-struct AtExitUnit {
-  AtExitCallback *callback = nullptr;
-  void *payload = nullptr;
-  constexpr AtExitUnit() = default;
-  constexpr AtExitUnit(AtExitCallback *c, void *p) : callback(c), payload(p) {}
-};
-
-#if defined(LIBC_TARGET_ARCH_IS_GPU)
-// The GPU build cannot handle the potentially recursive definitions required by
-// the BlockStore class. Additionally, the liklihood that someone exceeds this
-// while executing on the GPU is extremely small.
-// FIXME: It is not generally safe to use 'atexit' on the GPU because the
-//        mutexes simply passthrough. We will need a lock free stack.
-using ExitCallbackList = FixedVector<AtExitUnit, 64>;
-#elif defined(LIBC_COPT_PUBLIC_PACKAGING)
-using ExitCallbackList = ReverseOrderBlockStore<AtExitUnit, 32>;
-#else
-// BlockStore uses dynamic memory allocation. To avoid dynamic memory
-// allocation in tests, we use a fixed size callback list when built for
-// tests.
-// If we use BlockStore, then we will have to pull in malloc etc into
-// the tests. While this is not bad, the problem we have currently is
-// that LLVM libc' allocator is SCUDO. So, we will end up pulling SCUDO's
-// deps also (some of which are not yet available in LLVM libc) into the
-// integration tests.
-using ExitCallbackList = FixedVector<AtExitUnit, CALLBACK_LIST_SIZE_FOR_TESTS>;
-#endif // LIBC_COPT_PUBLIC_PACKAGING
-
-constinit ExitCallbackList exit_callbacks;
-
-void stdc_at_exit_func(void *payload) {
-  reinterpret_cast<StdCAtExitCallback *>(payload)();
-}
-
-void call_exit_callbacks() {
-  handler_list_mtx.lock();
-  while (!exit_callbacks.empty()) {
-    AtExitUnit &unit = exit_callbacks.back();
-    exit_callbacks.pop_back();
-    handler_list_mtx.unlock();
-    unit.callback(unit.payload);
-    handler_list_mtx.lock();
-  }
-  ExitCallbackList::destroy(&exit_callbacks);
-}
-
-int add_atexit_unit(const AtExitUnit &unit) {
-  cpp::lock_guard lock(handler_list_mtx);
-  if (exit_callbacks.push_back(unit))
-    return 0;
-  return -1;
-}
-
-} // namespace
-
 extern "C" {
 
-// TODO: Handle the last dso handle argument.
 int __cxa_atexit(AtExitCallback *callback, void *payload, void *) {
-  return add_atexit_unit({callback, payload});
+  return add_atexit_unit(atexit_callbacks, {callback, payload});
 }
 
-// TODO: Handle the dso handle argument. call_exit_callbacks should only invoke
-// the callbacks from this DSO. Requires adding support for __dso_handle.
 void __cxa_finalize(void *dso) {
   if (!dso)
-    call_exit_callbacks();
+    call_exit_callbacks(atexit_callbacks);
 }
 
 } // extern "C"
 
-LLVM_LIBC_FUNCTION(int, atexit, (StdCAtExitCallback * callback)) {
+LLVM_LIBC_FUNCTION(int, atexit, (__atexithandler_t callback)) {
   return add_atexit_unit(
+      atexit_callbacks,
       {&stdc_at_exit_func, reinterpret_cast<void *>(callback)});
 }
 
diff --git a/libc/src/stdlib/atexit.h b/libc/src/stdlib/atexit.h
index 7cf9d7c92191ddfa35df1092f5fc12937600f361..7faaf654247c2e8c1723d91a8fce4f100254b44c 100644
--- a/libc/src/stdlib/atexit.h
+++ b/libc/src/stdlib/atexit.h
@@ -9,13 +9,10 @@
 #ifndef LLVM_LIBC_SRC_STDLIB_ATEXIT_H
 #define LLVM_LIBC_SRC_STDLIB_ATEXIT_H
 
-#include <stddef.h> // For size_t
-
+#include "hdr/types/atexithandler_t.h"
 namespace LIBC_NAMESPACE {
 
-constexpr size_t CALLBACK_LIST_SIZE_FOR_TESTS = 1024;
-
-int atexit(void (*function)());
+int atexit(__atexithandler_t);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/stdlib/exit_handler.cpp b/libc/src/stdlib/exit_handler.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ed41247e4a31df5ea51990ff3b4a1f041ea42e92
--- /dev/null
+++ b/libc/src/stdlib/exit_handler.cpp
@@ -0,0 +1,42 @@
+//===--- Implementation of exit_handler------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/exit_handler.h"
+#include "src/__support/CPP/mutex.h" // lock_guard
+
+namespace LIBC_NAMESPACE {
+
+constinit ExitCallbackList at_quick_exit_callbacks;
+constinit ExitCallbackList atexit_callbacks;
+
+Mutex handler_list_mtx(false, false, false, false);
+
+void stdc_at_exit_func(void *payload) {
+  reinterpret_cast<StdCAtExitCallback *>(payload)();
+}
+
+void call_exit_callbacks(ExitCallbackList &callbacks) {
+  handler_list_mtx.lock();
+  while (!callbacks.empty()) {
+    AtExitUnit &unit = callbacks.back();
+    callbacks.pop_back();
+    handler_list_mtx.unlock();
+    unit.callback(unit.payload);
+    handler_list_mtx.lock();
+  }
+  ExitCallbackList::destroy(&callbacks);
+}
+
+int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit) {
+  cpp::lock_guard lock(handler_list_mtx);
+  if (callbacks.push_back(unit))
+    return 0;
+  return -1;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdlib/exit_handler.h b/libc/src/stdlib/exit_handler.h
new file mode 100644
index 0000000000000000000000000000000000000000..8494c2f2e526e94e82d81571ddf0a41ae208cab6
--- /dev/null
+++ b/libc/src/stdlib/exit_handler.h
@@ -0,0 +1,53 @@
+//===-- Implementation header for exit_handler ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H
+#define LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H
+
+#include "src/__support/CPP/mutex.h" // lock_guard
+#include "src/__support/blockstore.h"
+#include "src/__support/common.h"
+#include "src/__support/fixedvector.h"
+#include "src/__support/threads/mutex.h"
+
+namespace LIBC_NAMESPACE {
+
+using AtExitCallback = void(void *);
+using StdCAtExitCallback = void(void);
+constexpr size_t CALLBACK_LIST_SIZE_FOR_TESTS = 1024;
+
+struct AtExitUnit {
+  AtExitCallback *callback = nullptr;
+  void *payload = nullptr;
+  LIBC_INLINE constexpr AtExitUnit() = default;
+  LIBC_INLINE constexpr AtExitUnit(AtExitCallback *c, void *p)
+      : callback(c), payload(p) {}
+};
+
+#if defined(LIBC_TARGET_ARCH_IS_GPU)
+using ExitCallbackList = FixedVector<AtExitUnit, 64>;
+#elif defined(LIBC_COPT_PUBLIC_PACKAGING)
+using ExitCallbackList = ReverseOrderBlockStore<AtExitUnit, 32>;
+#else
+using ExitCallbackList = FixedVector<AtExitUnit, CALLBACK_LIST_SIZE_FOR_TESTS>;
+#endif
+
+extern ExitCallbackList atexit_callbacks;
+extern ExitCallbackList at_quick_exit_callbacks;
+
+extern Mutex handler_list_mtx;
+
+void stdc_at_exit_func(void *payload);
+
+void call_exit_callbacks(ExitCallbackList &callbacks);
+
+int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H
diff --git a/libc/src/stdlib/quick_exit.cpp b/libc/src/stdlib/quick_exit.cpp
index cf7f07bf2439a56b20155e1e903d474e2b86fe42..38f0a3db3e2cf0b5bf143b55cc532718e7f77b9d 100644
--- a/libc/src/stdlib/quick_exit.cpp
+++ b/libc/src/stdlib/quick_exit.cpp
@@ -9,13 +9,15 @@
 #include "src/stdlib/quick_exit.h"
 #include "src/__support/OSUtil/exit.h"
 #include "src/__support/common.h"
+#include "src/stdlib/exit_handler.h"
 
 // extern "C" void __cxa_finalize(void *);
-
 namespace LIBC_NAMESPACE {
 
+extern ExitCallbackList at_quick_exit_callbacks;
+
 [[noreturn]] LLVM_LIBC_FUNCTION(void, quick_exit, (int status)) {
-  // __cxa_finalize(nullptr);
+  call_exit_callbacks(at_quick_exit_callbacks);
   internal::exit(status);
 }
 
diff --git a/libc/src/sys/epoll/linux/CMakeLists.txt b/libc/src/sys/epoll/linux/CMakeLists.txt
index 4e661b262b85bfb9ded9e9c1cf47a624f9b8c5dc..5ba89bd1af603c5f1208e71565bc4395d48d9d98 100644
--- a/libc/src/sys/epoll/linux/CMakeLists.txt
+++ b/libc/src/sys/epoll/linux/CMakeLists.txt
@@ -48,6 +48,7 @@ add_entrypoint_object(
     libc.hdr.types.struct_timespec
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.macros.sanitizer
     libc.src.errno.errno
 )
 
@@ -65,6 +66,7 @@ add_entrypoint_object(
     libc.hdr.signal_macros
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.macros.sanitizer
     libc.src.errno.errno
 )
 
@@ -82,5 +84,6 @@ add_entrypoint_object(
     libc.hdr.signal_macros
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.macros.sanitizer
     libc.src.errno.errno
 )
diff --git a/libc/src/sys/epoll/linux/epoll_pwait.cpp b/libc/src/sys/epoll/linux/epoll_pwait.cpp
index 8f498d18d547dede411ce96e8fe84d06534e65a6..24b66f0721b30e8a447407ae3e79335868f5e2fc 100644
--- a/libc/src/sys/epoll/linux/epoll_pwait.cpp
+++ b/libc/src/sys/epoll/linux/epoll_pwait.cpp
@@ -13,6 +13,7 @@
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
@@ -33,6 +34,8 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait,
     return -1;
   }
 
+  MSAN_UNPOISON(events, ret * sizeof(struct epoll_event));
+
   return ret;
 }
 
diff --git a/libc/src/sys/epoll/linux/epoll_pwait2.cpp b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
index bd33cb6325cea8277d9e9427976e878f4a2b14e6..e13423a085a595e119a5fdb42153439dd4f1edce 100644
--- a/libc/src/sys/epoll/linux/epoll_pwait2.cpp
+++ b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
@@ -14,6 +14,7 @@
 #include "hdr/types/struct_timespec.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
@@ -35,6 +36,8 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait2,
     return -1;
   }
 
+  MSAN_UNPOISON(events, ret * sizeof(struct epoll_event));
+
   return ret;
 }
 
diff --git a/libc/src/sys/epoll/linux/epoll_wait.cpp b/libc/src/sys/epoll/linux/epoll_wait.cpp
index 95238d872d52fa607fea0f4bea6811a930dcdf17..3ce4a92e7969514c1edefbd26eefb9262fc82103 100644
--- a/libc/src/sys/epoll/linux/epoll_wait.cpp
+++ b/libc/src/sys/epoll/linux/epoll_wait.cpp
@@ -13,6 +13,7 @@
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
@@ -39,6 +40,8 @@ LLVM_LIBC_FUNCTION(int, epoll_wait,
     return -1;
   }
 
+  MSAN_UNPOISON(events, ret * sizeof(struct epoll_event));
+
   return ret;
 }
 
diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index 5e26a1000633b7a75fa39b04976df11c764277ad..b5c989aa1d433ccf27ea33762d5078b139e3efba 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -8,10 +8,14 @@ add_custom_target(libc-long-running-tests)
 
 add_subdirectory(UnitTest)
 
-if(LIBC_TARGET_OS_IS_GPU AND
-   (NOT TARGET libc.utils.gpu.loader OR LIBC_GPU_TESTS_DISABLED))
-  message(WARNING "Cannot build libc GPU tests, missing loader or architecture")
-  return()
+if(LIBC_TARGET_OS_IS_GPU)
+  if(NOT TARGET libc.utils.gpu.loader)
+    message(WARNING "Cannot build libc GPU tests, missing loader.")
+    return()
+  elseif(LIBC_GPU_TESTS_DISABLED)
+    message(WARNING "Cannot build libc GPU tests, missing target architecture.")
+    return()
+  endif()
 endif()
 
 add_subdirectory(include)
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 663aa2bb82cae803edb5be061cce0e9ea2857f58..d05377eca8a81cecb26ddee98aa8a483bb927fae 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -86,8 +86,8 @@ add_libc_test(
     libc.src.__support.uint128
 )
 
-# The GPU does not support varargs currently.
-if(NOT LIBC_TARGET_OS_IS_GPU)
+# NVPTX does not support varargs currently.
+if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
   add_libc_test(
     arg_list_test
     SUITE
@@ -132,6 +132,7 @@ add_libc_test(
   SRCS
     fixedvector_test.cpp
   DEPENDS
+    libc.src.__support.CPP.array
     libc.src.__support.fixedvector
 )
 
diff --git a/libc/test/src/__support/CPP/type_traits_test.cpp b/libc/test/src/__support/CPP/type_traits_test.cpp
index a2051f3802753263a46042a6a5da06b8cf938861..1c428e93a2fd8ace472926e2050d0bf128ab75d8 100644
--- a/libc/test/src/__support/CPP/type_traits_test.cpp
+++ b/libc/test/src/__support/CPP/type_traits_test.cpp
@@ -112,6 +112,15 @@ TEST(LlvmLibcTypeTraitsTest, add_rvalue_reference_void) {
                          const volatile void>));
 }
 
+TEST(LlvmLibcTypeTraitsTest, aligned_storage) {
+  struct S {
+    int a, b;
+  };
+  aligned_storage_t<sizeof(S), alignof(S)> buf;
+  EXPECT_EQ(alignof(buf), alignof(S));
+  EXPECT_EQ(sizeof(buf), sizeof(S));
+}
+
 TEST(LlvmLibcTypeTraitsTest, bool_constant) {
   EXPECT_TRUE((bool_constant<true>::value));
   EXPECT_FALSE((bool_constant<false>::value));
diff --git a/libc/test/src/__support/fixedvector_test.cpp b/libc/test/src/__support/fixedvector_test.cpp
index e9ffdd0203c2761c1954c672519e0c522049b2b9..212e1aed20f7cd2f34a9ab5b4e2bf1907bd568e3 100644
--- a/libc/test/src/__support/fixedvector_test.cpp
+++ b/libc/test/src/__support/fixedvector_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/array.h"
 #include "src/__support/fixedvector.h"
 #include "test/UnitTest/Test.h"
 
@@ -69,3 +70,29 @@ TEST(LlvmLibcFixedVectorTest, Iteration) {
   for (int &x : v)
     ASSERT_GE(x, 0);
 }
+
+TEST(LlvmLibcFixedVectorTest, ConstructionFromIterators) {
+  LIBC_NAMESPACE::cpp::array<int, 4> arr{1, 2, 3, 4};
+  LIBC_NAMESPACE::FixedVector<int, 5> vec(arr.begin(), arr.end());
+  ASSERT_EQ(vec.size(), arr.size());
+  for (size_t i = 0; i < arr.size(); ++i)
+    ASSERT_EQ(vec[i], arr[i]);
+}
+
+TEST(LlvmLibcFixedVectorTest, ConstructionFromCountAndValue) {
+  constexpr int kVal = 10;
+  LIBC_NAMESPACE::FixedVector<int, 5> vec(4, kVal);
+  ASSERT_EQ(vec.size(), size_t(4));
+  for (size_t i = 0; i < vec.size(); ++i)
+    ASSERT_EQ(vec[i], kVal);
+}
+
+TEST(LlvmLibcFixedVectorTest, ForwardIteration) {
+  LIBC_NAMESPACE::cpp::array<int, 4> arr{1, 2, 3, 4};
+  LIBC_NAMESPACE::FixedVector<int, 5> vec(arr.begin(), arr.end());
+  ASSERT_EQ(vec.size(), arr.size());
+  for (auto it = vec.begin(); it != vec.end(); ++it) {
+    auto idx = it - vec.begin();
+    ASSERT_EQ(*it, arr[idx]);
+  }
+}
diff --git a/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h b/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
index 504d1be94b891c1c3df40492f28606354ebed7dc..3027932c70f40525f19a0080d14b84a7e61ef34c 100644
--- a/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
+++ b/libc/test/src/math/performance_testing/BinaryOpSingleOutputPerf.h
@@ -9,6 +9,7 @@
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/src/math/performance_testing/Timer.h"
 
+#include <cstddef>
 #include <fstream>
 
 namespace LIBC_NAMESPACE {
@@ -25,7 +26,10 @@ public:
 
   static void run_perf_in_range(Func myFunc, Func otherFunc,
                                 StorageType startingBit, StorageType endingBit,
-                                StorageType N, std::ofstream &log) {
+                                size_t N, size_t rounds, std::ofstream &log) {
+    if (endingBit - startingBit < N)
+      N = endingBit - startingBit;
+
     auto runner = [=](Func func) {
       volatile T result;
       if (endingBit < startingBit) {
@@ -33,13 +37,15 @@ public:
       }
 
       StorageType step = (endingBit - startingBit) / N;
-      for (StorageType bitsX = startingBit, bitsY = endingBit;;
-           bitsX += step, bitsY -= step) {
-        T x = FPBits(bitsX).get_val();
-        T y = FPBits(bitsY).get_val();
-        result = func(x, y);
-        if (endingBit - bitsX < step) {
-          break;
+      for (size_t i = 0; i < rounds; i++) {
+        for (StorageType bitsX = startingBit, bitsY = endingBit;;
+             bitsX += step, bitsY -= step) {
+          T x = FPBits(bitsX).get_val();
+          T y = FPBits(bitsY).get_val();
+          result = func(x, y);
+          if (endingBit - bitsX < step) {
+            break;
+          }
         }
       }
     };
@@ -49,7 +55,7 @@ public:
     runner(myFunc);
     timer.stop();
 
-    double my_average = static_cast<double>(timer.nanoseconds()) / N;
+    double my_average = static_cast<double>(timer.nanoseconds()) / N / rounds;
     log << "-- My function --\n";
     log << "     Total time      : " << timer.nanoseconds() << " ns \n";
     log << "     Average runtime : " << my_average << " ns/op \n";
@@ -60,7 +66,8 @@ public:
     runner(otherFunc);
     timer.stop();
 
-    double other_average = static_cast<double>(timer.nanoseconds()) / N;
+    double other_average =
+        static_cast<double>(timer.nanoseconds()) / N / rounds;
     log << "-- Other function --\n";
     log << "     Total time      : " << timer.nanoseconds() << " ns \n";
     log << "     Average runtime : " << other_average << " ns/op \n";
@@ -71,22 +78,24 @@ public:
     log << "     Mine / Other's  : " << my_average / other_average << " \n";
   }
 
-  static void run_perf(Func myFunc, Func otherFunc, const char *logFile) {
+  static void run_perf(Func myFunc, Func otherFunc, int rounds,
+                       const char *logFile) {
     std::ofstream log(logFile);
     log << " Performance tests with inputs in denormal range:\n";
     run_perf_in_range(myFunc, otherFunc, /* startingBit= */ StorageType(0),
                       /* endingBit= */ FPBits::max_subnormal().uintval(),
-                      10'000'001, log);
+                      1'000'001, rounds, log);
     log << "\n Performance tests with inputs in normal range:\n";
     run_perf_in_range(myFunc, otherFunc,
                       /* startingBit= */ FPBits::min_normal().uintval(),
                       /* endingBit= */ FPBits::max_normal().uintval(),
-                      10'000'001, log);
+                      1'000'001, rounds, log);
     log << "\n Performance tests with inputs in normal range with exponents "
            "close to each other:\n";
-    run_perf_in_range(
-        myFunc, otherFunc, /* startingBit= */ FPBits(T(0x1.0p-10)).uintval(),
-        /* endingBit= */ FPBits(T(0x1.0p+10)).uintval(), 1'001'001, log);
+    run_perf_in_range(myFunc, otherFunc,
+                      /* startingBit= */ FPBits(T(0x1.0p-10)).uintval(),
+                      /* endingBit= */ FPBits(T(0x1.0p+10)).uintval(),
+                      1'000'001, rounds, log);
   }
 
   static void run_diff(Func myFunc, Func otherFunc, const char *logFile) {
@@ -117,6 +126,15 @@ public:
 #define BINARY_OP_SINGLE_OUTPUT_PERF(T, myFunc, otherFunc, filename)           \
   int main() {                                                                 \
     LIBC_NAMESPACE::testing::BinaryOpSingleOutputPerf<T>::run_perf(            \
-        &myFunc, &otherFunc, filename);                                        \
+        &myFunc, &otherFunc, 1, filename);                                     \
     return 0;                                                                  \
   }
+
+#define BINARY_OP_SINGLE_OUTPUT_PERF_EX(T, myFunc, otherFunc, rounds,          \
+                                        filename)                              \
+  {                                                                            \
+    LIBC_NAMESPACE::testing::BinaryOpSingleOutputPerf<T>::run_perf(            \
+        &myFunc, &otherFunc, rounds, filename);                                \
+    LIBC_NAMESPACE::testing::BinaryOpSingleOutputPerf<T>::run_perf(            \
+        &myFunc, &otherFunc, rounds, filename);                                \
+  }
diff --git a/libc/test/src/math/performance_testing/CMakeLists.txt b/libc/test/src/math/performance_testing/CMakeLists.txt
index d1fb24e37f7286bbb64fde9b60d7e152d99437a8..4ea78f9999e4d30073f32d6a8f2d34b728f6ad41 100644
--- a/libc/test/src/math/performance_testing/CMakeLists.txt
+++ b/libc/test/src/math/performance_testing/CMakeLists.txt
@@ -88,6 +88,8 @@ add_header_library(
   binary_op_single_output_diff
   HDRS
     BinaryOpSingleOutputPerf.h
+  DEPENDS
+    libc.src.__support.FPUtil.fp_bits
 )
 
 add_perf_binary(
@@ -343,6 +345,17 @@ add_perf_binary(
     -fno-builtin
 )
 
+add_perf_binary(
+  fmodf16_perf
+  SRCS
+    fmodf16_perf.cpp
+  DEPENDS
+    .binary_op_single_output_diff
+    libc.src.math.fmodf16
+    libc.src.__support.FPUtil.generic.fmod
+    libc.src.__support.macros.properties.types
+)
+
 add_perf_binary(
   fmodf128_perf
   SRCS
diff --git a/libc/test/src/math/performance_testing/fmodf16_perf.cpp b/libc/test/src/math/performance_testing/fmodf16_perf.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ff01fa6ca5870e8c7ea053fd7c437166f3ca7181
--- /dev/null
+++ b/libc/test/src/math/performance_testing/fmodf16_perf.cpp
@@ -0,0 +1,27 @@
+//===-- Performance test for fmodf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BinaryOpSingleOutputPerf.h"
+
+#include "src/__support/FPUtil/generic/FMod.h"
+#include "src/__support/macros/properties/types.h"
+
+#include <stdint.h>
+
+#define FMOD_FUNC(U) (LIBC_NAMESPACE::fputil::generic::FMod<float16, U>::eval)
+
+int main() {
+  BINARY_OP_SINGLE_OUTPUT_PERF_EX(float16, FMOD_FUNC(uint16_t),
+                                  FMOD_FUNC(uint32_t), 5000,
+                                  "fmodf16_u16_vs_u32_perf.log")
+
+  BINARY_OP_SINGLE_OUTPUT_PERF_EX(float16, FMOD_FUNC(uint16_t),
+                                  FMOD_FUNC(uint64_t), 5000,
+                                  "fmodf16_u16_vs_u64_perf.log")
+  return 0;
+}
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 09e54349501cabf7887ebe362ce7be837f4d3c14..84aa76c0a08810c86671b12285500f6256d85d2e 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -1091,6 +1091,18 @@ add_fp_unittest(
     libc.src.math.frexpl
 )
 
+add_fp_unittest(
+  frexpf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    frexpf16_test.cpp
+  HDRS
+    FrexpTest.h
+  DEPENDS
+    libc.src.math.frexpf16
+)
+
 add_fp_unittest(
   frexpf128_test
   SUITE
@@ -1353,7 +1365,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.ilogb
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1368,7 +1380,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.ilogbf
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1383,7 +1395,22 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.ilogbl
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
+add_fp_unittest(
+  ilogbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ilogbf16_test.cpp
+  HDRS
+    ILogbTest.h
+  DEPENDS
+    libc.src.math.ilogbf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1398,7 +1425,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.ilogbf128
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1413,7 +1440,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.llogb
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1428,7 +1455,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.llogbf
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1443,7 +1470,22 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.llogbl
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
+add_fp_unittest(
+  llogbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llogbf16_test.cpp
+  HDRS
+    ILogbTest.h
+  DEPENDS
+    libc.src.math.llogbf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1458,7 +1500,7 @@ add_fp_unittest(
     ILogbTest.h
   DEPENDS
     libc.src.math.llogbf128
-    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.manipulation_functions
 )
@@ -1529,8 +1571,11 @@ add_fp_unittest(
     libc-math-smoke-tests
   SRCS
     logb_test.cpp
+  HDRS
+    LogbTest.h
   DEPENDS
     libc.src.math.logb
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.manipulation_functions
 )
 
@@ -1540,8 +1585,11 @@ add_fp_unittest(
     libc-math-smoke-tests
   SRCS
     logbf_test.cpp
+  HDRS
+    LogbTest.h
   DEPENDS
     libc.src.math.logbf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.manipulation_functions
 )
 
@@ -1555,6 +1603,21 @@ add_fp_unittest(
     LogbTest.h
   DEPENDS
     libc.src.math.logbl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
+add_fp_unittest(
+  logbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    logbf16_test.cpp
+  HDRS
+    LogbTest.h
+  DEPENDS
+    libc.src.math.logbf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.manipulation_functions
 )
 
@@ -1564,8 +1627,11 @@ add_fp_unittest(
     libc-math-smoke-tests
   SRCS
     logbf128_test.cpp
+  HDRS
+    LogbTest.h
   DEPENDS
     libc.src.math.logbf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.manipulation_functions
 )
 
@@ -1579,6 +1645,7 @@ add_fp_unittest(
     ModfTest.h
   DEPENDS
     libc.src.math.modf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1593,6 +1660,7 @@ add_fp_unittest(
     ModfTest.h
   DEPENDS
     libc.src.math.modff
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1607,6 +1675,22 @@ add_fp_unittest(
     ModfTest.h
   DEPENDS
     libc.src.math.modfl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_fp_unittest(
+  modff16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    modff16_test.cpp
+  HDRS
+    ModfTest.h
+  DEPENDS
+    libc.src.math.modff16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1621,6 +1705,7 @@ add_fp_unittest(
     ModfTest.h
   DEPENDS
     libc.src.math.modff128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.nearest_integer_operations
 )
@@ -1705,6 +1790,7 @@ add_fp_unittest(
     FMinTest.h
   DEPENDS
     libc.src.math.fminf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1718,6 +1804,7 @@ add_fp_unittest(
     FMinTest.h
   DEPENDS
     libc.src.math.fmin
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1731,6 +1818,7 @@ add_fp_unittest(
     FMinTest.h
   DEPENDS
     libc.src.math.fminl
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1744,6 +1832,21 @@ add_fp_unittest(
     FMinTest.h
   DEPENDS
     libc.src.math.fminf128
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminf16_test.cpp
+  HDRS
+    FMinTest.h
+  DEPENDS
+    libc.src.math.fminf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1757,6 +1860,7 @@ add_fp_unittest(
     FMaxTest.h
   DEPENDS
     libc.src.math.fmaxf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1770,6 +1874,7 @@ add_fp_unittest(
     FMaxTest.h
   DEPENDS
     libc.src.math.fmax
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1783,6 +1888,7 @@ add_fp_unittest(
     FMaxTest.h
   DEPENDS
     libc.src.math.fmaxl
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1796,6 +1902,21 @@ add_fp_unittest(
     FMaxTest.h
   DEPENDS
     libc.src.math.fmaxf128
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaxf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaxf16_test.cpp
+  HDRS
+    FMaxTest.h
+  DEPENDS
+    libc.src.math.fmaxf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1809,6 +1930,21 @@ add_fp_unittest(
     FMaximumTest.h
   DEPENDS
     libc.src.math.fmaximuml
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximumf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximumf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximumf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1822,6 +1958,7 @@ add_fp_unittest(
     FMaximumTest.h
   DEPENDS
     libc.src.math.fmaximumf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1835,6 +1972,7 @@ add_fp_unittest(
     FMaximumTest.h
   DEPENDS
     libc.src.math.fmaximum
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1848,6 +1986,7 @@ add_fp_unittest(
     FMaximumTest.h
   DEPENDS
     libc.src.math.fmaximumf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1861,6 +2000,7 @@ add_fp_unittest(
     FMaximumNumTest.h
   DEPENDS
     libc.src.math.fmaximum_numf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1874,6 +2014,7 @@ add_fp_unittest(
     FMaximumNumTest.h
   DEPENDS
     libc.src.math.fmaximum_num
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1887,6 +2028,21 @@ add_fp_unittest(
     FMaximumNumTest.h
   DEPENDS
     libc.src.math.fmaximum_numl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_numf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_numf16_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_numf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1900,6 +2056,7 @@ add_fp_unittest(
     FMaximumNumTest.h
   DEPENDS
     libc.src.math.fmaximum_numf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1913,6 +2070,8 @@ add_fp_unittest(
     FMaximumMagTest.h
   DEPENDS
     libc.src.math.fmaximum_magf
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1926,6 +2085,8 @@ add_fp_unittest(
     FMaximumMagTest.h
   DEPENDS
     libc.src.math.fmaximum_mag
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1939,6 +2100,23 @@ add_fp_unittest(
     FMaximumMagTest.h
   DEPENDS
     libc.src.math.fmaximum_magl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_magf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_magf16_test.cpp
+  HDRS
+    FMaximumMagTest.h
+  DEPENDS
+    libc.src.math.fmaximum_magf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1952,10 +2130,11 @@ add_fp_unittest(
     FMaximumMagTest.h
   DEPENDS
     libc.src.math.fmaximum_magf128
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
-
 add_fp_unittest(
   fmaximum_mag_numf_test
   SUITE
@@ -1966,6 +2145,7 @@ add_fp_unittest(
     FMaximumMagNumTest.h
   DEPENDS
     libc.src.math.fmaximum_mag_numf
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1979,6 +2159,7 @@ add_fp_unittest(
     FMaximumMagNumTest.h
   DEPENDS
     libc.src.math.fmaximum_mag_num
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -1992,6 +2173,21 @@ add_fp_unittest(
     FMaximumMagNumTest.h
   DEPENDS
     libc.src.math.fmaximum_mag_numl
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_mag_numf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_numf16_test.cpp
+  HDRS
+    FMaximumMagNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_numf16
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2005,6 +2201,7 @@ add_fp_unittest(
     FMaximumMagNumTest.h
   DEPENDS
     libc.src.math.fmaximum_mag_numf128
+    libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2018,6 +2215,21 @@ add_fp_unittest(
     FMinimumTest.h
   DEPENDS
     libc.src.math.fminimuml
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimumf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimumf16_test.cpp
+  HDRS
+    FMinimumTest.h
+  DEPENDS
+    libc.src.math.fminimumf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2031,6 +2243,7 @@ add_fp_unittest(
     FMinimumTest.h
   DEPENDS
     libc.src.math.fminimumf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2044,6 +2257,7 @@ add_fp_unittest(
     FMinimumTest.h
   DEPENDS
     libc.src.math.fminimum
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2057,6 +2271,7 @@ add_fp_unittest(
     FMinimumTest.h
   DEPENDS
     libc.src.math.fminimumf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2070,6 +2285,7 @@ add_fp_unittest(
     FMinimumNumTest.h
   DEPENDS
     libc.src.math.fminimum_numf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2083,6 +2299,7 @@ add_fp_unittest(
     FMinimumNumTest.h
   DEPENDS
     libc.src.math.fminimum_num
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2096,6 +2313,21 @@ add_fp_unittest(
     FMinimumNumTest.h
   DEPENDS
     libc.src.math.fminimum_numl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_numf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_numf16_test.cpp
+  HDRS
+    FMinimumNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_numf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2109,6 +2341,7 @@ add_fp_unittest(
     FMinimumNumTest.h
   DEPENDS
     libc.src.math.fminimum_numf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2122,6 +2355,7 @@ add_fp_unittest(
     FMinimumMagTest.h
   DEPENDS
     libc.src.math.fminimum_magf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2135,6 +2369,7 @@ add_fp_unittest(
     FMinimumMagTest.h
   DEPENDS
     libc.src.math.fminimum_mag
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2148,6 +2383,21 @@ add_fp_unittest(
     FMinimumMagTest.h
   DEPENDS
     libc.src.math.fminimum_magl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_magf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_magf16_test.cpp
+  HDRS
+    FMinimumMagTest.h
+  DEPENDS
+    libc.src.math.fminimum_magf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2161,10 +2411,10 @@ add_fp_unittest(
     FMinimumMagTest.h
   DEPENDS
     libc.src.math.fminimum_magf128
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
-
 add_fp_unittest(
   fminimum_mag_numf_test
   SUITE
@@ -2175,6 +2425,7 @@ add_fp_unittest(
     FMinimumMagNumTest.h
   DEPENDS
     libc.src.math.fminimum_mag_numf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2188,6 +2439,7 @@ add_fp_unittest(
     FMinimumMagNumTest.h
   DEPENDS
     libc.src.math.fminimum_mag_num
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2201,6 +2453,21 @@ add_fp_unittest(
     FMinimumMagNumTest.h
   DEPENDS
     libc.src.math.fminimum_mag_numl
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_mag_numf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_numf16_test.cpp
+  HDRS
+    FMinimumMagNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_numf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2214,6 +2481,20 @@ add_fp_unittest(
     FMinimumMagNumTest.h
   DEPENDS
     libc.src.math.fminimum_mag_numf128
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmul_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmul_test.cpp
+  HDRS
+    FMulTest.h
+  DEPENDS
+    libc.src.math.fmul
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2331,6 +2612,20 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  remquof128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    remquof128_test.cpp
+  HDRS
+    RemQuoTest.h
+  DEPENDS
+    libc.src.math.remquof128
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   remquo_test
   SUITE
@@ -2426,6 +2721,22 @@ add_fp_unittest(
   UNIT_TEST_ONLY
 )
 
+add_fp_unittest(
+  nanf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nanf16_test.cpp
+  DEPENDS
+    libc.include.signal
+    libc.src.math.nanf16
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.macros.sanitizer
+  # FIXME: The nan tests currently have death tests, which aren't supported for
+  # hermetic tests.
+  UNIT_TEST_ONLY
+)
+
 add_fp_unittest(
   nanf128_test
   SUITE
@@ -2515,8 +2826,10 @@ add_fp_unittest(
   HDRS
     NextAfterTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.math.nextafter
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2529,8 +2842,10 @@ add_fp_unittest(
   HDRS
     NextAfterTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.math.nextafterf
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2543,8 +2858,26 @@ add_fp_unittest(
   HDRS
     NextAfterTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.math.nextafterl
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  nextafterf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextafterf16_test.cpp
+  HDRS
+    NextAfterTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.nextafterf16
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2557,8 +2890,10 @@ add_fp_unittest(
   HDRS
     NextAfterTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.math.nextafterf128
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2573,8 +2908,10 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
     HDRS
       NextTowardTest.h
     DEPENDS
+      libc.hdr.fenv_macros
       libc.src.math.nexttoward
-      libc.src.__support.FPUtil.basic_operations
+      libc.src.__support.CPP.bit
+      libc.src.__support.FPUtil.fenv_impl
       libc.src.__support.FPUtil.fp_bits
   )
 
@@ -2587,8 +2924,10 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
     HDRS
       NextTowardTest.h
     DEPENDS
+      libc.hdr.fenv_macros
       libc.src.math.nexttowardf
-      libc.src.__support.FPUtil.basic_operations
+      libc.src.__support.CPP.bit
+      libc.src.__support.FPUtil.fenv_impl
       libc.src.__support.FPUtil.fp_bits
   )
 endif()
@@ -2602,8 +2941,26 @@ add_fp_unittest(
   HDRS
     NextTowardTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.math.nexttowardl
-    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  nexttowardf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nexttowardf16_test.cpp
+  HDRS
+    NextTowardTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.nexttowardf16
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
 )
 
@@ -2617,7 +2974,6 @@ add_fp_unittest(
     NextDownTest.h
   DEPENDS
     libc.src.math.nextdown
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 add_fp_unittest(
@@ -2630,7 +2986,6 @@ add_fp_unittest(
     NextDownTest.h
   DEPENDS
     libc.src.math.nextdownf
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 add_fp_unittest(
@@ -2643,7 +2998,18 @@ add_fp_unittest(
     NextDownTest.h
   DEPENDS
     libc.src.math.nextdownl
-    libc.src.__support.FPUtil.manipulation_functions
+)
+
+add_fp_unittest(
+  nextdownf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextdownf16_test.cpp
+  HDRS
+    NextDownTest.h
+  DEPENDS
+    libc.src.math.nextdownf16
 )
 
 add_fp_unittest(
@@ -2656,7 +3022,6 @@ add_fp_unittest(
     NextDownTest.h
   DEPENDS
     libc.src.math.nextdownf128
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 add_fp_unittest(
@@ -2669,7 +3034,6 @@ add_fp_unittest(
     NextUpTest.h
   DEPENDS
     libc.src.math.nextup
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 add_fp_unittest(
@@ -2682,7 +3046,6 @@ add_fp_unittest(
     NextUpTest.h
   DEPENDS
     libc.src.math.nextupf
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 add_fp_unittest(
@@ -2695,7 +3058,18 @@ add_fp_unittest(
     NextUpTest.h
   DEPENDS
     libc.src.math.nextupl
-    libc.src.__support.FPUtil.manipulation_functions
+)
+
+add_fp_unittest(
+  nextupf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextupf16_test.cpp
+  HDRS
+    NextUpTest.h
+  DEPENDS
+    libc.src.math.nextupf16
 )
 
 add_fp_unittest(
@@ -2708,7 +3082,6 @@ add_fp_unittest(
     NextUpTest.h
   DEPENDS
     libc.src.math.nextupf128
-    libc.src.__support.FPUtil.manipulation_functions
 )
 
 # TODO(lntue): The current implementation of fputil::general::fma<float> is only
@@ -2866,10 +3239,10 @@ add_fp_unittest(
   HDRS
     FModTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.errno.errno
     libc.src.math.fmodf
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.FPUtil.fenv_impl
   # FIXME: Currently fails on the GPU build.
   UNIT_TEST_ONLY
 )
@@ -2883,10 +3256,10 @@ add_fp_unittest(
   HDRS
     FModTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.errno.errno
     libc.src.math.fmod
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.FPUtil.fenv_impl
   # FIXME: Currently fails on the GPU build.
   UNIT_TEST_ONLY
 )
@@ -2900,10 +3273,27 @@ add_fp_unittest(
   HDRS
     FModTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.errno.errno
     libc.src.math.fmodl
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.FPUtil.fenv_impl
+  # FIXME: Currently fails on the GPU build.
+  UNIT_TEST_ONLY
+)
+
+add_fp_unittest(
+  fmodf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmodf16_test.cpp
+  HDRS
+    FModTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.errno.errno
+    libc.src.math.fmodf16
+    libc.src.__support.FPUtil.fenv_impl
   # FIXME: Currently fails on the GPU build.
   UNIT_TEST_ONLY
 )
@@ -2917,10 +3307,10 @@ add_fp_unittest(
   HDRS
     FModTest.h
   DEPENDS
+    libc.hdr.fenv_macros
     libc.src.errno.errno
     libc.src.math.fmodf128
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.FPUtil.fenv_impl
   # FIXME: Currently fails on the GPU build.
   UNIT_TEST_ONLY
 )
diff --git a/libc/test/src/math/smoke/FMaxTest.h b/libc/test/src/math/smoke/FMaxTest.h
index df8e35e0bd16204ddce2270c01cb42a4cb823d91..f4c78b5d04b5b8ef13859a22a3d125dc925fca76 100644
--- a/libc/test/src/math/smoke/FMaxTest.h
+++ b/libc/test/src/math/smoke/FMaxTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -55,10 +56,11 @@ public:
   }
 
   void testRange(FMaxFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
diff --git a/libc/test/src/math/smoke/FMaximumMagNumTest.h b/libc/test/src/math/smoke/FMaximumMagNumTest.h
index aafb6d2b0d5eacab9bc70f161d1fbd4880b268d0..726f87059fc64bc5cdfcdc4a95aded27c48c7ca7 100644
--- a/libc/test/src/math/smoke/FMaximumMagNumTest.h
+++ b/libc/test/src/math/smoke/FMaximumMagNumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
@@ -68,10 +69,11 @@ public:
   }
 
   void testRange(FMaximumMagNumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -82,11 +84,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y))
         EXPECT_FP_EQ(x, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(y, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMaximumMagTest.h b/libc/test/src/math/smoke/FMaximumMagTest.h
index 7bb79a69be580f6b40d394ce43917dedf5ec27a5..b5b2c1ca79abc4ac3d51f95e14f0488e2087c160 100644
--- a/libc/test/src/math/smoke/FMaximumMagTest.h
+++ b/libc/test/src/math/smoke/FMaximumMagTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -56,10 +57,11 @@ public:
   }
 
   void testRange(FMaximumMagFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -70,11 +72,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y))
         EXPECT_FP_EQ(x, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(y, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMaximumNumTest.h b/libc/test/src/math/smoke/FMaximumNumTest.h
index da0ea2c247a9ee6feb5cb6cf1b963ce36217d237..ec7913509d39428b9754c30e30d86d766f8fe42a 100644
--- a/libc/test/src/math/smoke/FMaximumNumTest.h
+++ b/libc/test/src/math/smoke/FMaximumNumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -67,10 +68,11 @@ public:
   }
 
   void testRange(FMaximumNumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -81,11 +83,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (x > y) {
+      if (x > y)
         EXPECT_FP_EQ(x, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(y, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMaximumTest.h b/libc/test/src/math/smoke/FMaximumTest.h
index 1bd15163ed75291b1aeeacf7a149bda4d172a978..94e4a343190a5bde09cd3fa8f8998e08126c56f0 100644
--- a/libc/test/src/math/smoke/FMaximumTest.h
+++ b/libc/test/src/math/smoke/FMaximumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -55,10 +56,11 @@ public:
   }
 
   void testRange(FMaximumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -69,11 +71,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (x > y) {
+      if (x > y)
         EXPECT_FP_EQ(x, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(y, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMinTest.h b/libc/test/src/math/smoke/FMinTest.h
index f71b558cd3da2027b66e9e6d68a7e3d61ba5139e..629aaab729a86cc3761d52bd70068eb5b427c78f 100644
--- a/libc/test/src/math/smoke/FMinTest.h
+++ b/libc/test/src/math/smoke/FMinTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -55,10 +56,11 @@ public:
   }
 
   void testRange(FMinFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
diff --git a/libc/test/src/math/smoke/FMinimumMagNumTest.h b/libc/test/src/math/smoke/FMinimumMagNumTest.h
index e4b8fd9e335311e34475b2e8500eaf656c888d22..2ceca6ff95bac2af598e92d8ca7c8bcf77580132 100644
--- a/libc/test/src/math/smoke/FMinimumMagNumTest.h
+++ b/libc/test/src/math/smoke/FMinimumMagNumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
@@ -68,10 +69,11 @@ public:
   }
 
   void testRange(FMinimumMagNumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -82,11 +84,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y))
         EXPECT_FP_EQ(y, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(x, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMinimumMagTest.h b/libc/test/src/math/smoke/FMinimumMagTest.h
index 3e16622fe3fa41dc07254bb6a12e299195dd61dd..9c49446795ceef28fd1f728db1f85e2ee94d8f8d 100644
--- a/libc/test/src/math/smoke/FMinimumMagTest.h
+++ b/libc/test/src/math/smoke/FMinimumMagTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -56,10 +57,11 @@ public:
   }
 
   void testRange(FMinimumMagFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -70,11 +72,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (LIBC_NAMESPACE::fputil::abs(x) < LIBC_NAMESPACE::fputil::abs(y)) {
+      if (LIBC_NAMESPACE::fputil::abs(x) < LIBC_NAMESPACE::fputil::abs(y))
         EXPECT_FP_EQ(x, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(y, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMinimumNumTest.h b/libc/test/src/math/smoke/FMinimumNumTest.h
index 6186ea0df17cce1614c0ec1f52e5817ca88ac1a3..8004ee98745432d30044f4bff61a32fe365fcaf1 100644
--- a/libc/test/src/math/smoke/FMinimumNumTest.h
+++ b/libc/test/src/math/smoke/FMinimumNumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -67,10 +68,11 @@ public:
   }
 
   void testRange(FMinimumNumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -81,11 +83,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (x > y) {
+      if (x > y)
         EXPECT_FP_EQ(y, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(x, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FMinimumTest.h b/libc/test/src/math/smoke/FMinimumTest.h
index a267f6c78321452883de91c5986a587381412b47..242c857fbb99bc0fe1a16552f92cb83f3672ff57 100644
--- a/libc/test/src/math/smoke/FMinimumTest.h
+++ b/libc/test/src/math/smoke/FMinimumTest.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H
 
+#include "src/__support/CPP/algorithm.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -55,10 +56,11 @@ public:
   }
 
   void testRange(FMinimumFunc func) {
-    constexpr StorageType COUNT = 100'001;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
-         ++i, v += STEP, w -= STEP) {
+    constexpr int COUNT = 100'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0, w = STORAGE_MAX;
+    for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) {
       FPBits xbits(v), ybits(w);
       if (xbits.is_inf_or_nan())
         continue;
@@ -69,11 +71,10 @@ public:
       if ((x == 0) && (y == 0))
         continue;
 
-      if (x > y) {
+      if (x > y)
         EXPECT_FP_EQ(y, func(x, y));
-      } else {
+      else
         EXPECT_FP_EQ(x, func(x, y));
-      }
     }
   }
 };
diff --git a/libc/test/src/math/smoke/FModTest.h b/libc/test/src/math/smoke/FModTest.h
index f1015d6497fcd671f2b53e6ada5ce553402d2b95..405e3107438d42b15ee5d3a94e084f70b3fad09e 100644
--- a/libc/test/src/math/smoke/FModTest.h
+++ b/libc/test/src/math/smoke/FModTest.h
@@ -9,13 +9,13 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_FMODTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_FMODTEST_H
 
-#include "src/__support/FPUtil/BasicOperations.h"
-#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/errno/libc_errno.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include "hdr/math_macros.h"
+#include "hdr/fenv_macros.h"
 
 #define TEST_SPECIAL(x, y, expected, dom_err, expected_exception)              \
   EXPECT_FP_EQ(expected, f(x, y));                                             \
@@ -210,7 +210,8 @@ public:
   }
 
   void testRegularExtreme(FModFunc f) {
-
+    if constexpr (sizeof(T) < sizeof(float))
+      return;
     TEST_REGULAR(0x1p127L, 0x3p-149L, 0x1p-149L);
     TEST_REGULAR(0x1p127L, -0x3p-149L, 0x1p-149L);
     TEST_REGULAR(0x1p127L, 0x3p-148L, 0x1p-147L);
@@ -224,20 +225,20 @@ public:
     TEST_REGULAR(-0x1p127L, 0x3p-126L, -0x1p-125L);
     TEST_REGULAR(-0x1p127L, -0x3p-126L, -0x1p-125L);
 
-    if constexpr (sizeof(T) >= sizeof(double)) {
-      TEST_REGULAR(0x1p1023L, 0x3p-1074L, 0x1p-1073L);
-      TEST_REGULAR(0x1p1023L, -0x3p-1074L, 0x1p-1073L);
-      TEST_REGULAR(0x1p1023L, 0x3p-1073L, 0x1p-1073L);
-      TEST_REGULAR(0x1p1023L, -0x3p-1073L, 0x1p-1073L);
-      TEST_REGULAR(0x1p1023L, 0x3p-1022L, 0x1p-1021L);
-      TEST_REGULAR(0x1p1023L, -0x3p-1022L, 0x1p-1021L);
-      TEST_REGULAR(-0x1p1023L, 0x3p-1074L, -0x1p-1073L);
-      TEST_REGULAR(-0x1p1023L, -0x3p-1074L, -0x1p-1073L);
-      TEST_REGULAR(-0x1p1023L, 0x3p-1073L, -0x1p-1073L);
-      TEST_REGULAR(-0x1p1023L, -0x3p-1073L, -0x1p-1073L);
-      TEST_REGULAR(-0x1p1023L, 0x3p-1022L, -0x1p-1021L);
-      TEST_REGULAR(-0x1p1023L, -0x3p-1022L, -0x1p-1021L);
-    }
+    if constexpr (sizeof(T) < sizeof(double))
+      return;
+    TEST_REGULAR(0x1p1023L, 0x3p-1074L, 0x1p-1073L);
+    TEST_REGULAR(0x1p1023L, -0x3p-1074L, 0x1p-1073L);
+    TEST_REGULAR(0x1p1023L, 0x3p-1073L, 0x1p-1073L);
+    TEST_REGULAR(0x1p1023L, -0x3p-1073L, 0x1p-1073L);
+    TEST_REGULAR(0x1p1023L, 0x3p-1022L, 0x1p-1021L);
+    TEST_REGULAR(0x1p1023L, -0x3p-1022L, 0x1p-1021L);
+    TEST_REGULAR(-0x1p1023L, 0x3p-1074L, -0x1p-1073L);
+    TEST_REGULAR(-0x1p1023L, -0x3p-1074L, -0x1p-1073L);
+    TEST_REGULAR(-0x1p1023L, 0x3p-1073L, -0x1p-1073L);
+    TEST_REGULAR(-0x1p1023L, -0x3p-1073L, -0x1p-1073L);
+    TEST_REGULAR(-0x1p1023L, 0x3p-1022L, -0x1p-1021L);
+    TEST_REGULAR(-0x1p1023L, -0x3p-1022L, -0x1p-1021L);
   }
 };
 
diff --git a/libc/test/src/math/smoke/FMulTest.h b/libc/test/src/math/smoke/FMulTest.h
new file mode 100644
index 0000000000000000000000000000000000000000..33fb82c8d2da131ea8c5e00f40afe2fc36805678
--- /dev/null
+++ b/libc/test/src/math/smoke/FMulTest.h
@@ -0,0 +1,104 @@
+//===-- Utility class to test fmul[f|l] ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
+
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T, typename R>
+class FmulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMulFunc)(R, R);
+
+  void testMul(FMulFunc func) {
+
+    EXPECT_FP_EQ_ALL_ROUNDING(T(15.0), func(3.0, 5.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(T(0x1.0p-130), func(0x1.0p1, 0x1.0p-131));
+    EXPECT_FP_EQ_ALL_ROUNDING(T(0x1.0p-127), func(0x1.0p2, 0x1.0p-129));
+    EXPECT_FP_EQ_ALL_ROUNDING(T(1.0), func(1.0, 1.0));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(T(0.0), func(-0.0, -0.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(T(-0.0), func(0.0, -0.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(T(-0.0), func(-0.0, 0.0));
+
+    EXPECT_FP_EQ_ROUNDING_NEAREST(inf, func(0x1.0p100, 0x1.0p100));
+    EXPECT_FP_EQ_ROUNDING_UPWARD(inf, func(0x1.0p100, 0x1.0p100));
+    EXPECT_FP_EQ_ROUNDING_DOWNWARD(max_normal, func(0x1.0p100, 0x1.0p100));
+    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(max_normal, func(0x1.0p100, 0x1.0p100));
+
+    EXPECT_FP_EQ_ROUNDING_NEAREST(
+        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_DOWNWARD(
+        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(
+        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_UPWARD(
+        0x1p0, func(1.0, 1.0 + 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+
+    EXPECT_FP_EQ_ROUNDING_NEAREST(
+        0x1.0p-128f + 0x1.0p-148f,
+        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_UPWARD(
+        0x1.0p-128f + 0x1.0p-148f,
+        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_DOWNWARD(
+        0x1.0p-128f + 0x1.0p-149f,
+        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+    EXPECT_FP_EQ_ROUNDING_TOWARD_ZERO(
+        0x1.0p-128f + 0x1.0p-149f,
+        func(1.0, 0x1.0p-128 + 0x1.0p-149 + 0x1.0p-150));
+  }
+
+  void testSpecialInputs(FMulFunc func) {
+    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(inf, 0x1.0p-129));
+    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(0x1.0p-129, inf));
+    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(inf, 2.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(3.0, inf));
+    EXPECT_FP_EQ_ALL_ROUNDING(0.0, func(0.0, 0.0));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(neg_inf, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(aNaN, neg_inf));
+    EXPECT_FP_EQ_ALL_ROUNDING(inf, func(neg_inf, neg_inf));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, neg_inf));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(neg_inf, 0.0));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(neg_inf, 1.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(1.0, neg_inf));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(neg_inf, 0x1.0p-129));
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_inf, func(0x1.0p-129, neg_inf));
+
+    EXPECT_FP_EQ_ALL_ROUNDING(0.0, func(0.0, 0x1.0p-129));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, 0.0));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, inf));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(2.0, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0x1.0p-129, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(aNaN, aNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0.0, sNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(2.0, sNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(0x1.0p-129, sNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(inf, sNaN));
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, func(sNaN, sNaN));
+  }
+};
+
+#define LIST_FMUL_TESTS(T, R, func)                                            \
+  using LlvmLibcFmulTest = FmulTest<T, R>;                                     \
+  TEST_F(LlvmLibcFmulTest, Mul) { testMul(&func); }                            \
+  TEST_F(LlvmLibcFmulTest, NaNInf) { testSpecialInputs(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMULTEST_H
diff --git a/libc/test/src/math/smoke/FrexpTest.h b/libc/test/src/math/smoke/FrexpTest.h
index e9e496422f7326eebc46f7252ef4f93f33ec4b3b..fc2313a94ef09e8e6cb9c28813589f377f244307 100644
--- a/libc/test/src/math/smoke/FrexpTest.h
+++ b/libc/test/src/math/smoke/FrexpTest.h
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/__support/FPUtil/BasicOperations.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/libc/test/src/math/smoke/ILogbTest.h b/libc/test/src/math/smoke/ILogbTest.h
index 05f906b69947b23848cf071ebd212ba8e21a5239..3315ac2cbbc644ad1c8aaa3d8f83c5475f88c71f 100644
--- a/libc/test/src/math/smoke/ILogbTest.h
+++ b/libc/test/src/math/smoke/ILogbTest.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_ILOGBTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_ILOGBTEST_H
 
-#include "src/__support/CPP/limits.h" // INT_MAX
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/ManipulationFunctions.h"
 #include "test/UnitTest/FEnvSafeTest.h"
@@ -76,10 +76,12 @@ public:
   void test_subnormal_range(Func func) {
     constexpr StorageType MIN_SUBNORMAL = FPBits::min_subnormal().uintval();
     constexpr StorageType MAX_SUBNORMAL = FPBits::max_subnormal().uintval();
-    constexpr StorageType COUNT = 10'001;
-    constexpr StorageType STEP = (MAX_SUBNORMAL - MIN_SUBNORMAL) / COUNT;
+    constexpr int COUNT = 10'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>((MAX_SUBNORMAL - MIN_SUBNORMAL) / COUNT),
+        StorageType(1));
     for (StorageType v = MIN_SUBNORMAL; v <= MAX_SUBNORMAL; v += STEP) {
-      FPBits x_bits = FPBits(v);
+      FPBits x_bits(v);
       if (x_bits.is_zero() || x_bits.is_inf_or_nan())
         continue;
 
@@ -94,10 +96,12 @@ public:
   void test_normal_range(Func func) {
     constexpr StorageType MIN_NORMAL = FPBits::min_normal().uintval();
     constexpr StorageType MAX_NORMAL = FPBits::max_normal().uintval();
-    constexpr StorageType COUNT = 10'001;
-    constexpr StorageType STEP = (MAX_NORMAL - MIN_NORMAL) / COUNT;
+    constexpr int COUNT = 10'001;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>((MAX_NORMAL - MIN_NORMAL) / COUNT),
+        StorageType(1));
     for (StorageType v = MIN_NORMAL; v <= MAX_NORMAL; v += STEP) {
-      FPBits x_bits = FPBits(v);
+      FPBits x_bits(v);
       if (x_bits.is_zero() || x_bits.is_inf_or_nan())
         continue;
 
diff --git a/libc/test/src/math/smoke/LogbTest.h b/libc/test/src/math/smoke/LogbTest.h
index 4938fcf8f6f16eae3ea3aea4d20fc003f1e5ca32..0bb6e12665b9349a6700f24204dcec0c3b7f5ed4 100644
--- a/libc/test/src/math/smoke/LogbTest.h
+++ b/libc/test/src/math/smoke/LogbTest.h
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/ManipulationFunctions.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
@@ -69,10 +70,12 @@ public:
 
   void testRange(LogbFunc func) {
     using StorageType = typename FPBits::StorageType;
-    constexpr StorageType COUNT = 100'000;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
-      FPBits x_bits = FPBits(v);
+    constexpr int COUNT = 100'000;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0;
+    for (int i = 0; i <= COUNT; ++i, v += STEP) {
+      FPBits x_bits(v);
       if (x_bits.is_zero() || x_bits.is_inf_or_nan())
         continue;
 
diff --git a/libc/test/src/math/smoke/ModfTest.h b/libc/test/src/math/smoke/ModfTest.h
index 85db2d6d967b20d5a0a7bd80a00bade71d0009aa..6226e5d55f40cce4026c30e259b477bf6e0158b6 100644
--- a/libc/test/src/math/smoke/ModfTest.h
+++ b/libc/test/src/math/smoke/ModfTest.h
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "src/__support/FPUtil/NearestIntegerOperations.h"
 #include "test/UnitTest/FEnvSafeTest.h"
@@ -83,10 +84,12 @@ public:
   }
 
   void testRange(ModfFunc func) {
-    constexpr StorageType COUNT = 100'000;
-    constexpr StorageType STEP = STORAGE_MAX / COUNT;
-    for (StorageType i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
-      FPBits x_bits = FPBits(v);
+    constexpr int COUNT = 100'000;
+    constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max(
+        static_cast<StorageType>(STORAGE_MAX / COUNT), StorageType(1));
+    StorageType v = 0;
+    for (int i = 0; i <= COUNT; ++i, v += STEP) {
+      FPBits x_bits(v);
       if (x_bits.is_zero() || x_bits.is_inf_or_nan())
         continue;
 
diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h
index d65ccdf8e70c3ac5629d4479b225dc33c0c59f7c..6278f899d8a8b8b4abe22dd60b9f59ef943570e9 100644
--- a/libc/test/src/math/smoke/NextAfterTest.h
+++ b/libc/test/src/math/smoke/NextAfterTest.h
@@ -9,15 +9,15 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_NEXTAFTERTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_NEXTAFTERTEST_H
 
-#include "hdr/math_macros.h"
 #include "src/__support/CPP/bit.h"
-#include "src/__support/CPP/type_traits.h"
-#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include "hdr/fenv_macros.h"
+
 // TODO: Strengthen errno,exception checks and remove these assert macros
 // after new matchers/test fixtures are added
 #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception)      \
@@ -181,7 +181,7 @@ public:
     result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent());
     ASSERT_EQ(result_bits.get_mantissa(),
-              x_bits.get_mantissa() + StorageType(1));
+              static_cast<StorageType>(x_bits.get_mantissa() + StorageType(1)));
 
     x = -x;
 
@@ -195,7 +195,7 @@ public:
     result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent());
     ASSERT_EQ(result_bits.get_mantissa(),
-              x_bits.get_mantissa() + StorageType(1));
+              static_cast<StorageType>(x_bits.get_mantissa() + StorageType(1)));
   }
 };
 
diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h
index a24ec9ff6bd8169ac56a49905eb0498e047116c3..5992273d91901282e2cd36fab0fe08b9e563b950 100644
--- a/libc/test/src/math/smoke/NextTowardTest.h
+++ b/libc/test/src/math/smoke/NextTowardTest.h
@@ -9,16 +9,15 @@
 #ifndef LLVM_LIBC_TEST_SRC_MATH_NEXTTOWARDTEST_H
 #define LLVM_LIBC_TEST_SRC_MATH_NEXTTOWARDTEST_H
 
-#include "hdr/fenv_macros.h"
-#include "hdr/math_macros.h"
 #include "src/__support/CPP/bit.h"
-#include "src/__support/CPP/type_traits.h"
-#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include "hdr/fenv_macros.h"
+
 // TODO: Strengthen errno,exception checks and remove these assert macros
 // after new matchers/test fixtures are added
 #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception)      \
@@ -194,7 +193,7 @@ public:
     result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent());
     ASSERT_EQ(result_bits.get_mantissa(),
-              x_bits.get_mantissa() + StorageType(1));
+              static_cast<StorageType>(x_bits.get_mantissa() + StorageType(1)));
 
     x = -x;
 
@@ -208,7 +207,7 @@ public:
     result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent());
     ASSERT_EQ(result_bits.get_mantissa(),
-              x_bits.get_mantissa() + StorageType(1));
+              static_cast<StorageType>(x_bits.get_mantissa() + StorageType(1)));
   }
 };
 
diff --git a/libc/test/src/math/smoke/fmaxf16_test.cpp b/libc/test/src/math/smoke/fmaxf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..79c03b7685b06821ea83b209f6106f4fa33ae4fc
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaxf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaxf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaxTest.h"
+
+#include "src/math/fmaxf16.h"
+
+LIST_FMAX_TESTS(float16, LIBC_NAMESPACE::fmaxf16)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b11653eb395e2e7964439e5f37c4020461813cb1
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag_numf16 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/math/fmaximum_mag_numf16.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(float16, LIBC_NAMESPACE::fmaximum_mag_numf16)
diff --git a/libc/test/src/math/smoke/fmaximum_magf16_test.cpp b/libc/test/src/math/smoke/fmaximum_magf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6df1e4aaf6a967129ee6ebf72aa659614d0cd0df
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_magf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_magf16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/math/fmaximum_magf16.h"
+
+LIST_FMAXIMUM_MAG_TESTS(float16, LIBC_NAMESPACE::fmaximum_magf16)
diff --git a/libc/test/src/math/smoke/fmaximum_numf16_test.cpp b/libc/test/src/math/smoke/fmaximum_numf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7cb9cb068cf7f1c6e0bb5f663ee55ad2e7fccddd
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_numf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_numf16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/math/fmaximum_numf16.h"
+
+LIST_FMAXIMUM_NUM_TESTS(float16, LIBC_NAMESPACE::fmaximum_numf16)
diff --git a/libc/test/src/math/smoke/fmaximumf16_test.cpp b/libc/test/src/math/smoke/fmaximumf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4cbf846a1610f6fd0315a37e89fb2c608375d254
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximumf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximumf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/math/fmaximumf16.h"
+
+LIST_FMAXIMUM_TESTS(float16, LIBC_NAMESPACE::fmaximumf16)
diff --git a/libc/test/src/math/smoke/fminf16_test.cpp b/libc/test/src/math/smoke/fminf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4379a6e75065374c39d6b0fb53782cc9672f6c5e
--- /dev/null
+++ b/libc/test/src/math/smoke/fminf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminf16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinTest.h"
+
+#include "src/math/fminf16.h"
+
+LIST_FMIN_TESTS(float16, LIBC_NAMESPACE::fminf16)
diff --git a/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2c6aede529cd4620268f3d3abae1abf177cf3716
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag_numf16 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/math/fminimum_mag_numf16.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(float16, LIBC_NAMESPACE::fminimum_mag_numf16)
diff --git a/libc/test/src/math/smoke/fminimum_magf16_test.cpp b/libc/test/src/math/smoke/fminimum_magf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3687aecedfd9f28b5642f3cd19e856b101cdde74
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_magf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_magf16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/math/fminimum_magf16.h"
+
+LIST_FMINIMUM_MAG_TESTS(float16, LIBC_NAMESPACE::fminimum_magf16)
diff --git a/libc/test/src/math/smoke/fminimum_numf16_test.cpp b/libc/test/src/math/smoke/fminimum_numf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..67750816d229c3f595dad201a132f0c3cd46083a
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_numf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_numf16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/math/fminimum_numf16.h"
+
+LIST_FMINIMUM_NUM_TESTS(float16, LIBC_NAMESPACE::fminimum_numf16)
diff --git a/libc/test/src/math/smoke/fminimumf16_test.cpp b/libc/test/src/math/smoke/fminimumf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f8b0577490b9fe3611febda7ca8c5492d551b8a3
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimumf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimumf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/math/fminimumf16.h"
+
+LIST_FMINIMUM_TESTS(float16, LIBC_NAMESPACE::fminimumf16)
diff --git a/libc/test/src/math/smoke/fmodf16_test.cpp b/libc/test/src/math/smoke/fmodf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9a48c5aa0d6092910fc3c990b39857218e059a9d
--- /dev/null
+++ b/libc/test/src/math/smoke/fmodf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmodf16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FModTest.h"
+
+#include "src/math/fmodf16.h"
+
+LIST_FMOD_TESTS(float16, LIBC_NAMESPACE::fmodf16)
diff --git a/libc/test/src/math/smoke/fmul_test.cpp b/libc/test/src/math/smoke/fmul_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0eb664f7411eec6ec377221a5c8ef76e308000e6
--- /dev/null
+++ b/libc/test/src/math/smoke/fmul_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmul-------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMulTest.h"
+
+#include "src/math/fmul.h"
+
+LIST_FMUL_TESTS(float, double, LIBC_NAMESPACE::fmul)
diff --git a/libc/test/src/math/smoke/frexpf16_test.cpp b/libc/test/src/math/smoke/frexpf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4d5492c8cf0c021efe531984eaf7dc0e05864bc4
--- /dev/null
+++ b/libc/test/src/math/smoke/frexpf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for frexpf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FrexpTest.h"
+
+#include "src/math/frexpf16.h"
+
+LIST_FREXP_TESTS(float16, LIBC_NAMESPACE::frexpf16);
diff --git a/libc/test/src/math/smoke/ilogbf16_test.cpp b/libc/test/src/math/smoke/ilogbf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e046709523202e80bb4f5531f5cde4a17b79411a
--- /dev/null
+++ b/libc/test/src/math/smoke/ilogbf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ilogbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ILogbTest.h"
+
+#include "src/math/ilogbf16.h"
+
+LIST_INTLOGB_TESTS(int, float16, LIBC_NAMESPACE::ilogbf16);
diff --git a/libc/test/src/math/smoke/llogbf16_test.cpp b/libc/test/src/math/smoke/llogbf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8907681024b1b1d38c40ee3688bca8aba3acab42
--- /dev/null
+++ b/libc/test/src/math/smoke/llogbf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for llogbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ILogbTest.h"
+
+#include "src/math/llogbf16.h"
+
+LIST_INTLOGB_TESTS(long, float16, LIBC_NAMESPACE::llogbf16);
diff --git a/libc/test/src/math/smoke/logbf16_test.cpp b/libc/test/src/math/smoke/logbf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..cfc1a0521b3399f93dd6a33240151ad31f20f006
--- /dev/null
+++ b/libc/test/src/math/smoke/logbf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for logbf16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LogbTest.h"
+
+#include "src/math/logbf16.h"
+
+LIST_LOGB_TESTS(float16, LIBC_NAMESPACE::logbf16)
diff --git a/libc/test/src/math/smoke/modff16_test.cpp b/libc/test/src/math/smoke/modff16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..709337718230621586845794315e9f266d4cf0ff
--- /dev/null
+++ b/libc/test/src/math/smoke/modff16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for modff16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModfTest.h"
+
+#include "src/math/modff16.h"
+
+LIST_MODF_TESTS(float16, LIBC_NAMESPACE::modff16)
diff --git a/libc/test/src/math/smoke/nanf16_test.cpp b/libc/test/src/math/smoke/nanf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ec17a73d881aa62ab840b1587554e83424538427
--- /dev/null
+++ b/libc/test/src/math/smoke/nanf16_test.cpp
@@ -0,0 +1,51 @@
+//===-- Unittests for nanf16 ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/macros/sanitizer.h"
+#include "src/math/nanf16.h"
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include <signal.h>
+
+class LlvmLibcNanf16Test : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+public:
+  using StorageType = LIBC_NAMESPACE::fputil::FPBits<float16>::StorageType;
+
+  void run_test(const char *input_str, StorageType bits) {
+    float16 result = LIBC_NAMESPACE::nanf16(input_str);
+    auto actual_fp = LIBC_NAMESPACE::fputil::FPBits<float16>(result);
+    auto expected_fp = LIBC_NAMESPACE::fputil::FPBits<float16>(bits);
+    EXPECT_EQ(actual_fp.uintval(), expected_fp.uintval());
+  };
+};
+
+TEST_F(LlvmLibcNanf16Test, NCharSeq) {
+  run_test("", 0x7e00);
+  run_test("123", 0x7e7b);
+  run_test("0x123", 0x7f23);
+  run_test("1a", 0x7e00);
+  run_test("1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_",
+           0x7e00);
+  run_test("10000000000000000000000000000000000000000000000000", 0x7e00);
+}
+
+TEST_F(LlvmLibcNanf16Test, RandomString) {
+  run_test(" 1234", 0x7e00);
+  run_test("-1234", 0x7e00);
+  run_test("asd&f", 0x7e00);
+  run_test("123 ", 0x7e00);
+}
+
+#ifndef LIBC_HAVE_ADDRESS_SANITIZER
+TEST_F(LlvmLibcNanf16Test, InvalidInput) {
+  EXPECT_DEATH([] { LIBC_NAMESPACE::nanf16(nullptr); }, WITH_SIGNAL(SIGSEGV));
+}
+#endif // LIBC_HAVE_ADDRESS_SANITIZER
diff --git a/libc/test/src/math/smoke/nextafterf16_test.cpp b/libc/test/src/math/smoke/nextafterf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..860a0c74acbca94a7b2d6788db316c501b7bc2aa
--- /dev/null
+++ b/libc/test/src/math/smoke/nextafterf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for nextafterf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextAfterTest.h"
+
+#include "src/math/nextafterf16.h"
+
+LIST_NEXTAFTER_TESTS(float16, LIBC_NAMESPACE::nextafterf16)
diff --git a/libc/test/src/math/smoke/nextdownf16_test.cpp b/libc/test/src/math/smoke/nextdownf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..353f0858617783965f1a30905baadaebf3e23fed
--- /dev/null
+++ b/libc/test/src/math/smoke/nextdownf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for nextdownf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextDownTest.h"
+
+#include "src/math/nextdownf16.h"
+
+LIST_NEXTDOWN_TESTS(float16, LIBC_NAMESPACE::nextdownf16)
diff --git a/libc/test/src/math/smoke/nexttowardf16_test.cpp b/libc/test/src/math/smoke/nexttowardf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8490e8de94ceeb0e9bd143666cf54462a4c14e12
--- /dev/null
+++ b/libc/test/src/math/smoke/nexttowardf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for nexttowardf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextTowardTest.h"
+
+#include "src/math/nexttowardf16.h"
+
+LIST_NEXTTOWARD_TESTS(float16, LIBC_NAMESPACE::nexttowardf16)
diff --git a/libc/test/src/math/smoke/nextupf16_test.cpp b/libc/test/src/math/smoke/nextupf16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a146d279f3a719ef5636b35256c7cf20e67aed56
--- /dev/null
+++ b/libc/test/src/math/smoke/nextupf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for nextupf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextUpTest.h"
+
+#include "src/math/nextupf16.h"
+
+LIST_NEXTUP_TESTS(float16, LIBC_NAMESPACE::nextupf16)
diff --git a/libc/test/src/math/smoke/remquof128_test.cpp b/libc/test/src/math/smoke/remquof128_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8ef6c3b31cef2b3b82eb157c01c0f81b0d4d57e9
--- /dev/null
+++ b/libc/test/src/math/smoke/remquof128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for remquof128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RemQuoTest.h"
+
+#include "src/math/remquof128.h"
+
+LIST_REMQUO_TESTS(float128, LIBC_NAMESPACE::remquof128)
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index 6a7faedece380ac55681fd80aa653af6292090dd..38488778c657c33f87774517f6efa5bf762e7d93 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -354,7 +354,20 @@ if(LLVM_LIBC_FULL_BUILD)
       libc.src.stdlib.exit
       libc.src.stdlib.atexit
       libc.src.__support.CPP.array
-      libc.src.__support.CPP.utility
+  )
+
+  add_libc_test(
+    at_quick_exit_test
+    # The EXPECT_EXITS test is only availible for unit tests.
+    UNIT_TEST_ONLY
+    SUITE
+      libc-stdlib-tests
+    SRCS
+      at_quick_exit_test.cpp
+    DEPENDS
+      libc.src.stdlib.quick_exit
+      libc.src.stdlib.at_quick_exit
+      libc.src.__support.CPP.array
   )
 
   add_libc_test(
diff --git a/libc/test/src/stdlib/at_quick_exit_test.cpp b/libc/test/src/stdlib/at_quick_exit_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e0a258d9fb2d96a2d9b3f2a5080e3e26eba3edde
--- /dev/null
+++ b/libc/test/src/stdlib/at_quick_exit_test.cpp
@@ -0,0 +1,90 @@
+//===-- Unittests for at_quick_exit ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/utility.h"
+#include "src/stdlib/at_quick_exit.h"
+#include "src/stdlib/quick_exit.h"
+#include "test/UnitTest/Test.h"
+
+static int a;
+TEST(LlvmLibcAtQuickExit, Basic) {
+  // In case tests ever run multiple times.
+  a = 0;
+
+  auto test = [] {
+    int status = LIBC_NAMESPACE::at_quick_exit(+[] {
+      if (a != 1)
+        __builtin_trap();
+    });
+    status |= LIBC_NAMESPACE::at_quick_exit(+[] { a++; });
+    if (status)
+      __builtin_trap();
+
+    LIBC_NAMESPACE::quick_exit(0);
+  };
+  EXPECT_EXITS(test, 0);
+}
+
+TEST(LlvmLibcAtQuickExit, AtQuickExitCallsSysExit) {
+  auto test = [] {
+    LIBC_NAMESPACE::at_quick_exit(+[] { _Exit(1); });
+    LIBC_NAMESPACE::quick_exit(0);
+  };
+  EXPECT_EXITS(test, 1);
+}
+
+static int size;
+static LIBC_NAMESPACE::cpp::array<int, 256> arr;
+
+template <int... Ts>
+void register_at_quick_exit_handlers(
+    LIBC_NAMESPACE::cpp::integer_sequence<int, Ts...>) {
+  (LIBC_NAMESPACE::at_quick_exit(+[] { arr[size++] = Ts; }), ...);
+}
+
+template <int count> constexpr auto get_test() {
+  return [] {
+    LIBC_NAMESPACE::at_quick_exit(+[] {
+      if (size != count)
+        __builtin_trap();
+      for (int i = 0; i < count; i++)
+        if (arr[i] != count - 1 - i)
+          __builtin_trap();
+    });
+    register_at_quick_exit_handlers(
+        LIBC_NAMESPACE::cpp::make_integer_sequence<int, count>{});
+    LIBC_NAMESPACE::quick_exit(0);
+  };
+}
+
+TEST(LlvmLibcAtQuickExit, ReverseOrder) {
+  // In case tests ever run multiple times.
+  size = 0;
+
+  auto test = get_test<32>();
+  EXPECT_EXITS(test, 0);
+}
+
+TEST(LlvmLibcAtQuickExit, Many) {
+  // In case tests ever run multiple times.
+  size = 0;
+
+  auto test = get_test<256>();
+  EXPECT_EXITS(test, 0);
+}
+
+TEST(LlvmLibcAtQuickExit, HandlerCallsAtQuickExit) {
+  auto test = [] {
+    LIBC_NAMESPACE::at_quick_exit(+[] {
+      LIBC_NAMESPACE::at_quick_exit(+[] { LIBC_NAMESPACE::quick_exit(1); });
+    });
+    LIBC_NAMESPACE::quick_exit(0);
+  };
+  EXPECT_EXITS(test, 1);
+}
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 5ce17952430854158d8a8adec6055d939a99c41a..9858ae905983f329c26d86734a613dfae975c9e7 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -198,7 +198,7 @@ set( tahiti_aliases pitcairn verde oland hainan bonaire kabini kaveri hawaii
   gfx1010 gfx1011 gfx1012 gfx1013
   gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036
   gfx1100 gfx1101 gfx1102 gfx1103
-  gfx1150 gfx1151
+  gfx1150 gfx1151 gfx1152
   gfx1200 gfx1201
 )
 
diff --git a/libcxx/.clang-tidy b/libcxx/.clang-tidy
index ec7cab9b878ed9efa746fb1c042aa76d97ef32e4..d0b4cced9ffc38e9a2e4456a94e894b65fe8fa08 100644
--- a/libcxx/.clang-tidy
+++ b/libcxx/.clang-tidy
@@ -57,8 +57,12 @@ CheckOptions:
     value: _
   - key:   readability-identifier-naming.TemplateParameterIgnoredRegexp
     value: (.*\:auto|expr-type) # This is https://llvm.org/PR56464
-  - key:   readability-identifier-naming.ValueTemplateParameterIgnoredRegexp # TODO: enforce naming of variable parameters
-    value: .*
+  - key:   readability-identifier-naming.ValueTemplateParameterCase
+    value: CamelCase
+  - key:   readability-identifier-naming.ValueTemplateParameterPrefix
+    value: _
+  - key:   readability-identifier-naming.ValueTemplateParameterIgnoredRegexp
+    value: (__[a-z_]|_[A-Z]).* # TODO: Converge on a single style for value template parameters
 
 # TODO: investigate these checks
 # bugprone-branch-clone,
diff --git a/libcxx/docs/Hardening.rst b/libcxx/docs/Hardening.rst
index 0761f42368e9211b0113a80070ba6a3cfb4485a0..996e7aed881abac0fca854b846536d6025e9ef1b 100644
--- a/libcxx/docs/Hardening.rst
+++ b/libcxx/docs/Hardening.rst
@@ -1,4 +1,4 @@
-.. _hardening-modes:
+.. _hardening:
 
 ===============
 Hardening Modes
@@ -29,8 +29,11 @@ modes are:
   rigour impacts performance more than fast mode: we recommend benchmarking to
   determine if that is acceptable for your program.
 - **Debug mode**, which enables all the available checks in the library,
-  including internal assertions, some of which might be very expensive. This
-  mode is intended to be used for testing, not in production.
+  including heuristic checks that might have significant performance overhead as
+  well as internal library assertions. This mode should be used in
+  non-production environments (such as test suites, CI, or local development).
+  We don’t commit to a particular level of performance in this mode and it’s
+  *not* intended to be used in production.
 
 .. note::
 
@@ -72,17 +75,367 @@ to control the level by passing **one** of the following options to the compiler
 Notes for vendors
 -----------------
 
-Vendors can set the default hardening mode by providing ``LIBCXX_HARDENING_MODE``
-as a configuration option, with the possible values of ``none``, ``fast``,
-``extensive`` and ``debug``. The default value is ``none`` which doesn't enable
-any hardening checks (this mode is sometimes called the ``unchecked`` mode).
+Vendors can set the default hardening mode by providing
+``LIBCXX_HARDENING_MODE`` as a configuration option, with the possible values of
+``none``, ``fast``, ``extensive`` and ``debug``. The default value is ``none``
+which doesn't enable any hardening checks (this mode is sometimes called the
+``unchecked`` mode).
 
 This option controls both the hardening mode that the precompiled library is
 built with and the default hardening mode that users will build with. If set to
 ``none``, the precompiled library will not contain any assertions, and user code
 will default to building without assertions.
 
-Iterator bounds checking
-------------------------
+Vendors can also override the way the program is terminated when an assertion
+fails by :ref:`providing a custom header <override-assertion-handler>`.
 
-TODO(hardening)
+Assertion categories
+====================
+
+Inside the library, individual assertions are grouped into different
+*categories*. Each hardening mode enables a different set of assertion
+categories; categories provide an additional layer of abstraction that makes it
+easier to reason about the high-level semantics of a hardening mode.
+
+.. note::
+
+  Users are not intended to interact with these categories directly -- the
+  categories are considered internal to the library and subject to change.
+
+- ``valid-element-access`` -- checks that any attempts to access a container
+  element, whether through the container object or through an iterator, are
+  valid and do not attempt to go out of bounds or otherwise access
+  a non-existent element. This also includes operations that set up an imminent
+  invalid access (e.g. incrementing an end iterator). For iterator checks to
+  work, bounded iterators must be enabled in the ABI. Types like
+  ``std::optional`` and ``std::function`` are considered containers (with at
+  most one element) for the purposes of this check.
+
+- ``valid-input-range`` -- checks that ranges (whether expressed as an iterator
+  pair, an iterator and a sentinel, an iterator and a count, or
+  a ``std::range``) given as input to library functions are valid:
+  - the sentinel is reachable from the begin iterator;
+  - TODO(hardening): both iterators refer to the same container.
+
+  ("input" here refers to "an input given to an algorithm", not to an iterator
+  category)
+
+  Violating assertions in this category leads to an out-of-bounds access.
+
+- ``non-null`` -- checks that the pointer being dereferenced is not null. On
+  most modern platforms, the zero address does not refer to an actual location
+  in memory, so a null pointer dereference would not compromise the memory
+  security of a program (however, it is still undefined behavior that can result
+  in strange errors due to compiler optimizations).
+
+- ``non-overlapping-ranges`` -- for functions that take several ranges as
+  arguments, checks that those ranges do not overlap.
+
+- ``valid-deallocation`` -- checks that an attempt to deallocate memory is valid
+  (e.g. the given object was allocated by the given allocator). Violating this
+  category typically results in a memory leak.
+
+- ``valid-external-api-call`` -- checks that a call to an external API doesn't
+  fail in an unexpected manner. This includes triggering documented cases of
+  undefined behavior in an external library (like attempting to unlock an
+  unlocked mutex in pthreads). Any API external to the library falls under this
+  category (from system calls to compiler intrinsics). We generally don't expect
+  these failures to compromise memory safety or otherwise create an immediate
+  security issue.
+
+- ``compatible-allocator`` -- checks any operations that exchange nodes between
+  containers to make sure the containers have compatible allocators.
+
+- ``argument-within-domain`` -- checks that the given argument is within the
+  domain of valid arguments for the function. Violating this typically produces
+  an incorrect result (e.g. ``std::clamp`` returns the original value without
+  clamping it due to incorrect functors) or puts an object into an invalid state
+  (e.g. a string view where only a subset of elements is accessible). This
+  category is for assertions violating which doesn't cause any immediate issues
+  in the library -- whatever the consequences are, they will happen in the user
+  code.
+
+- ``pedantic`` -- checks preconditions that are imposed by the Standard, but
+  violating which happens to be benign in libc++.
+
+- ``semantic-requirement`` -- checks that the given argument satisfies the
+  semantic requirements imposed by the Standard. Typically, there is no simple
+  way to completely prove that a semantic requirement is satisfied; thus, this
+  would often be a heuristic check and it might be quite expensive.
+
+- ``internal`` -- checks that internal invariants of the library hold. These
+  assertions don't depend on user input.
+
+- ``uncategorized`` -- for assertions that haven't been properly classified yet.
+  This category is an escape hatch used for some existing assertions in the
+  library; all new code should have its assertions properly classified.
+
+Mapping between the hardening modes and the assertion categories
+================================================================
+
+.. list-table::
+    :header-rows: 1
+    :widths: auto
+
+    * - Category name
+      - ``fast``
+      - ``extensive``
+      - ``debug``
+    * - ``valid-element-access``
+      - ✅
+      - ✅
+      - ✅
+    * - ``valid-input-range``
+      - ✅
+      - ✅
+      - ✅
+    * - ``non-null``
+      - ❌
+      - ✅
+      - ✅
+    * - ``non-overlapping-ranges``
+      - ❌
+      - ✅
+      - ✅
+    * - ``valid-deallocation``
+      - ❌
+      - ✅
+      - ✅
+    * - ``valid-external-api-call``
+      - ❌
+      - ✅
+      - ✅
+    * - ``compatible-allocator``
+      - ❌
+      - ✅
+      - ✅
+    * - ``argument-within-domain``
+      - ❌
+      - ✅
+      - ✅
+    * - ``pedantic``
+      - ❌
+      - ✅
+      - ✅
+    * - ``semantic-requirement``
+      - ❌
+      - ❌
+      - ✅
+    * - ``internal``
+      - ❌
+      - ❌
+      - ✅
+    * - ``uncategorized``
+      - ❌
+      - ✅
+      - ✅
+
+.. note::
+
+  At the moment, each subsequent hardening mode is a strict superset of the
+  previous one (in other words, each subsequent mode only enables additional
+  assertion categories without disabling any), but this won't necessarily be
+  true for any hardening modes that might be added in the future.
+
+.. note::
+
+  The categories enabled by each mode are subject to change and users should not
+  rely on the precise assertions enabled by a mode at a given point in time.
+  However, the library does guarantee to keep the hardening modes stable and
+  to fulfill the semantics documented here.
+
+Hardening assertion failure
+===========================
+
+In production modes (``fast`` and ``extensive``), a hardening assertion failure
+immediately ``_traps <https://llvm.org/docs/LangRef.html#llvm-trap-intrinsic>``
+the program. This is the safest approach that also minimizes the code size
+penalty as the failure handler maps to a single instruction. The downside is
+that the failure provides no additional details other than the stack trace
+(which might also be affected by optimizations).
+
+TODO(hardening): describe ``__builtin_verbose_trap`` once we can use it.
+
+In the ``debug`` mode, an assertion failure terminates the program in an
+unspecified manner and also outputs the associated error message to the error
+output. This is less secure and increases the size of the binary (among other
+things, it has to store the error message strings) but makes the failure easier
+to debug. It also allows testing the error messages in our test suite.
+
+.. _override-assertion-handler:
+
+Overriding the assertion failure handler
+----------------------------------------
+
+Vendors can override the default assertion handler mechanism by following these
+steps:
+
+- create a header file that provides a definition of a macro called
+  ``_LIBCPP_ASSERTION_HANDLER``. The macro will be invoked when a hardening
+  assertion fails, with a single parameter containing a null-terminated string
+  with the error message.
+- when configuring the library, provide the path to custom header (relative to
+  the root of the repository) via the CMake variable
+  ``LIBCXX_ASSERTION_HANDLER_FILE``.
+
+Note that almost all libc++ headers include the assertion handler header which
+means it should not include anything non-trivial from the standard library to
+avoid creating circular dependencies.
+
+There is no existing mechanism for users to override the assertion handler
+because the ability to do the override other than at configure-time carries an
+unavoidable code size penalty that would otherwise be imposed on all users,
+whether they require such customization or not. Instead, we let vendors decide
+what's right on their platform for their users -- a vendor who wishes to provide
+this capability is free to do so, e.g. by declaring the assertion handler as an
+overridable function.
+
+ABI
+===
+
+Setting a hardening mode does **not** affect the ABI. Each mode uses the subset
+of checks available in the current ABI configuration which is determined by the
+platform.
+
+It is important to stress that whether a particular check is enabled depends on
+the combination of the selected hardening mode and the hardening-related ABI
+options. Some checks require changing the ABI from the "default" to store
+additional information in the library classes -- e.g. checking whether an
+iterator is valid upon dereference generally requires storing data about bounds
+inside the iterator object. Using ``std::span`` as an example, setting the
+hardening mode to ``fast`` will always enable the ``valid-element-access``
+checks when accessing elements via a ``std::span`` object, but whether
+dereferencing a ``std::span`` iterator does the equivalent check depends on the
+ABI configuration.
+
+ABI options
+-----------
+
+Vendors can use the following ABI options to enable additional hardening checks:
+
+- ``_LIBCPP_ABI_BOUNDED_ITERATORS`` -- changes the iterator type of select
+  containers (see below) to a bounded iterator that keeps track of whether it's
+  within the bounds of the original container and asserts valid bounds on every
+  dereference.
+
+  ABI impact: changes the iterator type of the relevant containers.
+
+  Supported containers:
+
+  - ``span``;
+  - ``string_view``.
+
+ABI tags
+--------
+
+We use ABI tags to allow translation units built with different hardening modes
+to interact with each other without causing ODR violations. Knowing how
+hardening modes are encoded into the ABI tags might be useful to examine
+a binary and determine whether it was built with hardening enabled.
+
+.. warning::
+  We don't commit to the encoding scheme used by the ABI tags being stable
+  between different releases of libc++. The tags themselves are never stable, by
+  design -- new releases increase the version number. The following describes
+  the state of the latest release and is for informational purposes only.
+
+The first character of an ABI tag encodes the hardening mode:
+
+- ``f`` -- [f]ast mode;
+- ``s`` -- extensive ("[s]afe") mode;
+- ``d`` -- [d]ebug mode;
+- ``n`` -- [n]one mode.
+
+Hardened containers status
+==========================
+
+.. list-table::
+    :header-rows: 1
+    :widths: auto
+
+    * - Name
+      - Member functions
+      - Iterators (ABI-dependent)
+    * - ``span``
+      - ✅
+      - ✅
+    * - ``string_view``
+      - ✅
+      - ✅
+    * - ``array``
+      - ✅
+      - ❌
+    * - ``vector``
+      - ✅
+      - ❌
+    * - ``string``
+      - ✅
+      - ❌
+    * - ``list``
+      - ✅
+      - ❌
+    * - ``forward_list``
+      - ❌
+      - ❌
+    * - ``deque``
+      - ✅
+      - ❌
+    * - ``map``
+      - ❌
+      - ❌
+    * - ``set``
+      - ❌
+      - ❌
+    * - ``multimap``
+      - ❌
+      - ❌
+    * - ``multiset``
+      - ❌
+      - ❌
+    * - ``unordered_map``
+      - Partial
+      - Partial
+    * - ``unordered_set``
+      - Partial
+      - Partial
+    * - ``unordered_multimap``
+      - Partial
+      - Partial
+    * - ``unordered_multiset``
+      - Partial
+      - Partial
+    * - ``mdspan``
+      - ✅
+      - ❌
+    * - ``optional``
+      - ✅
+      - N/A
+    * - ``function``
+      - ❌
+      - N/A
+    * - ``variant``
+      - N/A
+      - N/A
+    * - ``any``
+      - N/A
+      - N/A
+    * - ``expected``
+      - ✅
+      - N/A
+    * - ``valarray``
+      - Partial
+      - N/A
+    * - ``bitset``
+      - ❌
+      - N/A
+
+Testing
+=======
+
+Please see :ref:`Testing documentation <testing-hardening-assertions>`.
+
+Further reading
+===============
+
+- ``_Hardening RFC <https://discourse.llvm.org/t/rfc-hardening-in-libc/73925>``:
+  contains some of the design rationale.
diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst
index fcd630e09b449f69d218ec49a29f130c93713a56..4f7b9b362e5e66a33b1c73b577161e7bcd902521 100644
--- a/libcxx/docs/ReleaseNotes/18.rst
+++ b/libcxx/docs/ReleaseNotes/18.rst
@@ -40,7 +40,7 @@ and C++26 features.
 
 New hardened modes for the library have been added, replacing the legacy debug mode that was
 removed in the LLVM 17 release. Unlike the legacy debug mode, some of these hardening modes are
-also intended to be used in production. See :ref:`hardening-modes` for more details.
+also intended to be used in production. See :ref:`hardening` for more details.
 
 Work on the ranges support has progressed. See
 :ref:`ranges-status` for the current status.
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 0bc343acd281c1d78d58e1c6cd89ebb75ba1fbf1..6ab4cafd2bb768278466751c28be284a1c6bceca 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -43,7 +43,6 @@ Implemented Papers
 - P2819R2 - Add ``tuple`` protocol to ``complex``
 - P2495R3 - Interfacing ``stringstream``\s with ``string_view``
 - P2867R2 - Remove Deprecated ``strstream``\s From C++26
-- P2869R4 - Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26
 - P2872R3 - Remove ``wstring_convert`` From C++26
 - P3142R0 - Printing Blank Lines with ``println`` (as DR against C++23)
 - P2944R3 - Comparisons for ``reference_wrapper`` (comparison operators for ``reference_wrapper`` only)
@@ -69,9 +68,6 @@ Improvements and New Features
 
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM`` macro has been added to make the declarations in ``<strstream>`` available.
 
-- The ``_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS`` macro has been added to make the declarations in ``<memory>``
-  available.
-
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT`` macro has been added to make the declarations in ``<locale>``
   available.
 
diff --git a/libcxx/docs/Status/Cxx20.rst b/libcxx/docs/Status/Cxx20.rst
index b08b99394fbb7e512b8973c06981a3af1afd1ebd..c00d6fb23728692b77ce2351a66947d073d22ce0 100644
--- a/libcxx/docs/Status/Cxx20.rst
+++ b/libcxx/docs/Status/Cxx20.rst
@@ -58,8 +58,6 @@ Paper Status
       * ``GPS clock`` not done
       * ``UTC clock`` not done
 
-   .. [#note-P0718] P0718: Implemented deprecation of ``shared_ptr`` atomic access APIs only.
-
 .. _issues-status-cxx20:
 
 Library Working Group Issues Status
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 6598cd18358f3e59f221009131575954575e97d0..462c7eff55d589d07e6f987fe7abfe1f5ae469d1 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -12,7 +12,7 @@
 "`P0600R1 <https://wg21.link/P0600R1>`__","LWG","nodiscard in the Library","Albuquerque","|Complete|","16.0"
 "`P0616R0 <https://wg21.link/P0616R0>`__","LWG","de-pessimize legacy <numeric> algorithms with std::move","Albuquerque","|Complete|","12.0"
 "`P0653R2 <https://wg21.link/P0653R2>`__","LWG","Utility to convert a pointer to a raw pointer","Albuquerque","|Complete|","6.0"
-"`P0718R2 <https://wg21.link/P0718R2>`__","LWG","Atomic shared_ptr","Albuquerque","|Partial| [#note-P0718]_",""
+"`P0718R2 <https://wg21.link/P0718R2>`__","LWG","Atomic shared_ptr","Albuquerque","",""
 "`P0767R1 <https://wg21.link/P0767R1>`__","CWG","Deprecate POD","Albuquerque","|Complete|","7.0"
 "`P0768R1 <https://wg21.link/P0768R1>`__","CWG","Library Support for the Spaceship (Comparison) Operator","Albuquerque","|Complete|",""
 "`P0777R1 <https://wg21.link/P0777R1>`__","LWG","Treating Unnecessary ``decay``\ ","Albuquerque","|Complete|","7.0"
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 30a601858b63a6d010f65879981ac55c71268dab..ea060f18acdeeae927fcae084ecde82d8061fb97 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -48,7 +48,7 @@
 "","","","","","",""
 "`P2875R4 <https://wg21.link/P2875R4>`__","LWG","Undeprecate ``polymorphic_allocator::destroy`` for C++26","Tokyo March 2024","|Complete|","15.0",""
 "`P2867R2 <https://wg21.link/P2867R2>`__","LWG","Remove Deprecated ``strstreams`` From C++26","Tokyo March 2024","|Complete|","19.0",""
-"`P2869R4 <https://wg21.link/P2869R4>`__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","|Complete|","19.0",""
+"`P2869R4 <https://wg21.link/P2869R4>`__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","","",""
 "`P2872R3 <https://wg21.link/P2872R3>`__","LWG","Remove ``wstring_convert`` From C++26","Tokyo March 2024","|Complete|","19.0",""
 "`P3107R5 <https://wg21.link/P3107R5>`__","LWG","Permit an efficient implementation of ``std::print``","Tokyo March 2024","","","|format| |DR|"
 "`P3142R0 <https://wg21.link/P3142R0>`__","LWG","Printing Blank Lines with ``println``","Tokyo March 2024","|Complete| [#note-P3142R0]_","19.0","|format|"
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index 50ee9d4ee400b45a784ee7dca56d5e6091c7d9ed..d9f4fe467fe362e9272105fe38bafc3a009fd1a0 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -480,3 +480,48 @@ For example:
   $ ./algorithms.libcxx.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks
 
 For more information about running benchmarks see `Google Benchmark`_.
+
+
+.. _testing-hardening-assertions:
+
+Testing hardening assertions
+============================
+
+Each hardening assertion should be tested using death tests (via the
+``TEST_LIBCPP_ASSERT_FAILURE`` macro). Use the ``libcpp-hardening-mode`` Lit
+feature to make sure the assertion is enabled in (and only in) the intended
+modes. The convention is to use `assert.` in the name of the test file to make
+it easier to identify as a hardening test, e.g. ``assert.my_func.pass.cpp``.
+A toy example:
+
+.. code-block:: cpp
+
+  // Note: the following three annotations are currently needed to use the
+  // `TEST_LIBCPP_ASSERT_FAILURE`.
+  // REQUIRES: has-unix-headers
+  // UNSUPPORTED: c++03
+  // XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+  // Example: only run this test in `fast`/`extensive`/`debug` modes.
+  // UNSUPPORTED: libcpp-hardening-mode=none
+  // Example: only run this test in the `debug` mode.
+  // REQUIRES: libcpp-hardening-mode=debug
+  // Example: only run this test in `extensive`/`debug` modes.
+  // REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+
+  #include <header_being_tested>
+
+  #include "check_assertion.h" // Contains the `TEST_LIBCPP_ASSERT_FAILURE` macro
+
+  int main(int, char**) {
+    std::type_being_tested foo;
+    int bad_input = -1;
+    TEST_LIBCPP_ASSERT_FAILURE(foo.some_function_that_asserts(bad_input),
+        "The expected assertion message");
+
+    return 0;
+  }
+
+Note that error messages are only tested (matched) if the ``debug``
+hardening mode is used.
+
diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst
index 0fdaeb433ac6a0cff79a8933239b02709d3ed7bb..df08875c13beae9f0cb096745069bad14c847123 100644
--- a/libcxx/docs/UsingLibcxx.rst
+++ b/libcxx/docs/UsingLibcxx.rst
@@ -252,14 +252,10 @@ C++26 Specific Configuration Macros
 **_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM**:
   This macro is used to re-enable all named declarations in ``<strstream>``.
 
-**_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS**:
-  This macro is used to re-enable all ``shared_ptr`` atomic access APIs in ``<memory>``.
-
 **_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT**:
   This macro is used to re-enable the ``wstring_convert`` and ``wbuffer_convert``
   in ``<locale>``.
 
-
 Libc++ Extensions
 =================
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index cfe1f44777bcac70d2d9f175e5d1040b0f40046e..cbbcc74e21b443b60074f4c4f61348ed817c807b 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -250,6 +250,7 @@ set(files
   __chrono/convert_to_tm.h
   __chrono/day.h
   __chrono/duration.h
+  __chrono/exception.h
   __chrono/file_clock.h
   __chrono/formatter.h
   __chrono/hh_mm_ss.h
diff --git a/libcxx/include/__chrono/exception.h b/libcxx/include/__chrono/exception.h
new file mode 100644
index 0000000000000000000000000000000000000000..75fd0615b7e08127bbf66973431f67d15b9258ed
--- /dev/null
+++ b/libcxx/include/__chrono/exception.h
@@ -0,0 +1,129 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
+
+#ifndef _LIBCPP___CHRONO_EXCEPTION_H
+#define _LIBCPP___CHRONO_EXCEPTION_H
+
+#include <version>
+// Enable the contents of the header only when libc++ was built with experimental features enabled.
+#if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#  include <__chrono/calendar.h>
+#  include <__chrono/local_info.h>
+#  include <__chrono/time_point.h>
+#  include <__config>
+#  include <__configuration/availability.h>
+#  include <__verbose_abort>
+#  include <format>
+#  include <stdexcept>
+#  include <string>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  if _LIBCPP_STD_VER >= 20
+
+namespace chrono {
+
+class nonexistent_local_time : public runtime_error {
+public:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI nonexistent_local_time(const local_time<_Duration>& __time, const local_info& __info)
+      : runtime_error{__create_message(__time, __info)} {
+    // [time.zone.exception.nonexist]/2
+    //   Preconditions: i.result == local_info::nonexistent is true.
+    // The value of __info.result is not used.
+    _LIBCPP_ASSERT_PEDANTIC(__info.result == local_info::nonexistent,
+                            "creating an nonexistent_local_time from a local_info that is not non-existent");
+  }
+
+  _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI ~nonexistent_local_time() override; // exported as key function
+
+private:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI string __create_message(const local_time<_Duration>& __time, const local_info& __info) {
+    return std::format(
+        R"({} is in a gap between
+{} {} and
+{} {} which are both equivalent to
+{} UTC)",
+        __time,
+        local_seconds{__info.first.end.time_since_epoch()} + __info.first.offset,
+        __info.first.abbrev,
+        local_seconds{__info.second.begin.time_since_epoch()} + __info.second.offset,
+        __info.second.abbrev,
+        __info.first.end);
+  }
+};
+
+template <class _Duration>
+_LIBCPP_NORETURN _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_nonexistent_local_time(
+    [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw nonexistent_local_time(__time, __info);
+#    else
+  _LIBCPP_VERBOSE_ABORT("nonexistent_local_time was thrown in -fno-exceptions mode");
+#    endif
+}
+
+class ambiguous_local_time : public runtime_error {
+public:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI ambiguous_local_time(const local_time<_Duration>& __time, const local_info& __info)
+      : runtime_error{__create_message(__time, __info)} {
+    // [time.zone.exception.ambig]/2
+    //   Preconditions: i.result == local_info::ambiguous is true.
+    // The value of __info.result is not used.
+    _LIBCPP_ASSERT_PEDANTIC(__info.result == local_info::ambiguous,
+                            "creating an ambiguous_local_time from a local_info that is not ambiguous");
+  }
+
+  _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI ~ambiguous_local_time() override; // exported as key function
+
+private:
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI string __create_message(const local_time<_Duration>& __time, const local_info& __info) {
+    return std::format(
+        // There are two spaces after the full-stop; this has been verified
+        // in the sources of the Standard.
+        R"({0} is ambiguous.  It could be
+{0} {1} == {2} UTC or
+{0} {3} == {4} UTC)",
+        __time,
+        __info.first.abbrev,
+        __time - __info.first.offset,
+        __info.second.abbrev,
+        __time - __info.second.offset);
+  }
+};
+
+template <class _Duration>
+_LIBCPP_NORETURN _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_ambiguous_local_time(
+    [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw ambiguous_local_time(__time, __info);
+#    else
+  _LIBCPP_VERBOSE_ABORT("ambiguous_local_time was thrown in -fno-exceptions mode");
+#    endif
+}
+
+} // namespace chrono
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
+
+#endif // _LIBCPP___CHRONO_EXCEPTION_H
diff --git a/libcxx/include/__chrono/time_zone.h b/libcxx/include/__chrono/time_zone.h
index 91ddab8903fe219b235825dacaa08ee036302f17..de11dac1eef0c2d4aaf2db87a5b21911e40acb3c 100644
--- a/libcxx/include/__chrono/time_zone.h
+++ b/libcxx/include/__chrono/time_zone.h
@@ -16,12 +16,16 @@
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
 #if !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
+#  include <__chrono/calendar.h>
 #  include <__chrono/duration.h>
+#  include <__chrono/exception.h>
+#  include <__chrono/local_info.h>
 #  include <__chrono/sys_info.h>
 #  include <__chrono/system_clock.h>
 #  include <__compare/strong_order.h>
 #  include <__config>
 #  include <__memory/unique_ptr.h>
+#  include <__type_traits/common_type.h>
 #  include <string_view>
 
 #  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -38,6 +42,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace chrono {
 
+enum class choose { earliest, latest };
+
 class _LIBCPP_AVAILABILITY_TZDB time_zone {
   _LIBCPP_HIDE_FROM_ABI time_zone() = default;
 
@@ -63,12 +69,91 @@ public:
     return __get_info(chrono::time_point_cast<seconds>(__time));
   }
 
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI local_info get_info(const local_time<_Duration>& __time) const {
+    return __get_info(chrono::time_point_cast<seconds>(__time));
+  }
+
+  // We don't apply nodiscard here since this function throws on many inputs,
+  // so it could be used as a validation.
+  template <class _Duration>
+  _LIBCPP_HIDE_FROM_ABI sys_time<common_type_t<_Duration, seconds>> to_sys(const local_time<_Duration>& __time) const {
+    local_info __info = get_info(__time);
+    switch (__info.result) {
+    case local_info::unique:
+      return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+    case local_info::nonexistent:
+      chrono::__throw_nonexistent_local_time(__time, __info);
+
+    case local_info::ambiguous:
+      chrono::__throw_ambiguous_local_time(__time, __info);
+    }
+
+    // TODO TZDB The Standard does not specify anything in these cases.
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -1, "cannot convert the local time; it would be before the minimum system clock value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -2, "cannot convert the local time; it would be after the maximum system clock value");
+
+    return {};
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI sys_time<common_type_t<_Duration, seconds>>
+  to_sys(const local_time<_Duration>& __time, choose __z) const {
+    local_info __info = get_info(__time);
+    switch (__info.result) {
+    case local_info::unique:
+    case local_info::nonexistent: // first and second are the same
+      return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+    case local_info::ambiguous:
+      switch (__z) {
+      case choose::earliest:
+        return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.first.offset};
+
+      case choose::latest:
+        return sys_time<common_type_t<_Duration, seconds>>{__time.time_since_epoch() - __info.second.offset};
+
+        // Note a value out of bounds is not specified.
+      }
+    }
+
+    // TODO TZDB The standard does not specify anything in these cases.
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -1, "cannot convert the local time; it would be before the minimum system clock value");
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.result != -2, "cannot convert the local time; it would be after the maximum system clock value");
+
+    return {};
+  }
+
+  template <class _Duration>
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI local_time<common_type_t<_Duration, seconds>>
+  to_local(const sys_time<_Duration>& __time) const {
+    using _Dp = common_type_t<_Duration, seconds>;
+
+    sys_info __info = get_info(__time);
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.offset >= chrono::seconds{0} || __time.time_since_epoch() >= _Dp::min() - __info.offset,
+        "cannot convert the system time; it would be before the minimum local clock value");
+
+    _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
+        __info.offset <= chrono::seconds{0} || __time.time_since_epoch() <= _Dp::max() - __info.offset,
+        "cannot convert the system time; it would be after the maximum local clock value");
+
+    return local_time<_Dp>{__time.time_since_epoch() + __info.offset};
+  }
+
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI const __impl& __implementation() const noexcept { return *__impl_; }
 
 private:
   [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI string_view __name() const noexcept;
 
   [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI sys_info __get_info(sys_seconds __time) const;
+  [[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI local_info __get_info(local_seconds __time) const;
 
   unique_ptr<__impl> __impl_;
 };
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index 17aceb042f524a042ccba8a3826e28bfff9868b2..73375fa47d07fd6af7b6d0c7e33ec43cfe960426 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -127,8 +127,7 @@
 //
 // Supported containers:
 // - `span`;
-// - `string_view`;
-// - `array`.
+// - `string_view`.
 // #define _LIBCPP_ABI_BOUNDED_ITERATORS
 
 #if defined(_LIBCPP_COMPILER_CLANG_BASED)
diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h
index 8a0e3b7ffa58404a9a7a7e77db612f5f89f2fc5b..385dabaee424de9007add133e77f2443a4a82c8a 100644
--- a/libcxx/include/__functional/bind.h
+++ b/libcxx/include/__functional/bind.h
@@ -95,7 +95,7 @@ __mu(_Ti& __ti, tuple<_Uj...>& __uj) {
   return std::__mu_expand(__ti, __uj, __indices());
 }
 
-template <bool IsPh, class _Ti, class _Uj>
+template <bool _IsPh, class _Ti, class _Uj>
 struct __mu_return2 {};
 
 template <class _Ti, class _Uj>
@@ -120,7 +120,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ti& __mu(_Ti& __ti,
   return __ti;
 }
 
-template <class _Ti, bool IsReferenceWrapper, bool IsBindEx, bool IsPh, class _TupleUj>
+template <class _Ti, bool _IsReferenceWrapper, bool _IsBindEx, bool _IsPh, class _TupleUj>
 struct __mu_return_impl;
 
 template <bool _Invokable, class _Ti, class... _Uj>
diff --git a/libcxx/include/__mdspan/extents.h b/libcxx/include/__mdspan/extents.h
index f6bcd940ee6077d36663121846478643da833212..16fac13549614c69be4e2a00f15484c0859c9807 100644
--- a/libcxx/include/__mdspan/extents.h
+++ b/libcxx/include/__mdspan/extents.h
@@ -137,9 +137,9 @@ private:
   // static mapping of indices to the position in the dynamic values array
   using _DynamicIdxMap = __static_partial_sums<static_cast<size_t>(_Values == _DynTag)...>;
 
-  template <size_t... Indices>
-  _LIBCPP_HIDE_FROM_ABI static constexpr _DynamicValues __zeros(index_sequence<Indices...>) noexcept {
-    return _DynamicValues{((void)Indices, 0)...};
+  template <size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _DynamicValues __zeros(index_sequence<_Indices...>) noexcept {
+    return _DynamicValues{((void)_Indices, 0)...};
   }
 
 public:
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index de5707c4a67b0c7b40c690b004b5777e4e27a7b8..00db96185be7c650159e85932f529b25f4756c79 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -1583,15 +1583,13 @@ private:
 
 _LIBCPP_EXPORTED_FROM_ABI __sp_mut& __get_sp_mut(const void*);
 
-#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
-
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) {
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) {
   return false;
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   shared_ptr<_Tp> __q = *__p;
@@ -1600,13 +1598,12 @@ _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(co
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
-atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) {
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) {
   return std::atomic_load(__p);
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+_LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   __p->swap(__r);
@@ -1614,14 +1611,12 @@ _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
-atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
+inline _LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
   std::atomic_store(__p, __r);
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
-atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   __p->swap(__r);
@@ -1630,13 +1625,13 @@ atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
 atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
   return std::atomic_exchange(__p, __r);
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_HIDE_FROM_ABI bool
 atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
   shared_ptr<_Tp> __temp;
   __sp_mut& __m = std::__get_sp_mut(__p);
@@ -1654,25 +1649,23 @@ atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, share
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool
+inline _LIBCPP_HIDE_FROM_ABI bool
 atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
   return std::atomic_compare_exchange_strong(__p, __v, __w);
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
     shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
   return std::atomic_compare_exchange_strong(__p, __v, __w);
 }
 
 template <class _Tp>
-_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
+inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
     shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
   return std::atomic_compare_exchange_weak(__p, __v, __w);
 }
 
-#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
-
 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/datasizeof.h b/libcxx/include/__type_traits/datasizeof.h
index 54fde242ebcde1946a5401a895e6d64b5d3e89c9..35c12921e8ffa197d9559c6c28cf5d20f9f54560 100644
--- a/libcxx/include/__type_traits/datasizeof.h
+++ b/libcxx/include/__type_traits/datasizeof.h
@@ -26,7 +26,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_extension(datasizeof)
+#if __has_keyword(__datasizeof) || __has_extension(datasizeof)
 template <class _Tp>
 inline const size_t __datasizeof_v = __datasizeof(_Tp);
 #else
diff --git a/libcxx/include/__type_traits/promote.h b/libcxx/include/__type_traits/promote.h
index e22b4a422c2c8049f45b48d5e0b566fa495376e2..2b2a6843b9150219166884e613ae5dafb059b7a5 100644
--- a/libcxx/include/__type_traits/promote.h
+++ b/libcxx/include/__type_traits/promote.h
@@ -11,8 +11,12 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/is_same.h>
-#include <__utility/declval.h>
+#include <__type_traits/is_arithmetic.h>
+
+#if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER == 1700
+#  include <__type_traits/is_same.h>
+#  include <__utility/declval.h>
+#endif
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -20,6 +24,34 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+// TODO(LLVM-20): Remove this workaround
+#if !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER != 1700
+
+template <class... _Args>
+class __promote {
+  static_assert((is_arithmetic<_Args>::value && ...));
+
+  static float __test(float);
+  static double __test(char);
+  static double __test(int);
+  static double __test(unsigned);
+  static double __test(long);
+  static double __test(unsigned long);
+  static double __test(long long);
+  static double __test(unsigned long long);
+#  ifndef _LIBCPP_HAS_NO_INT128
+  static double __test(__int128_t);
+  static double __test(__uint128_t);
+#  endif
+  static double __test(double);
+  static long double __test(long double);
+
+public:
+  using type = decltype((__test(_Args()) + ...));
+};
+
+#else
+
 template <class _Tp>
 struct __numeric_type {
   static void __test(...);
@@ -31,10 +63,10 @@ struct __numeric_type {
   static double __test(unsigned long);
   static double __test(long long);
   static double __test(unsigned long long);
-#ifndef _LIBCPP_HAS_NO_INT128
+#  ifndef _LIBCPP_HAS_NO_INT128
   static double __test(__int128_t);
   static double __test(__uint128_t);
-#endif
+#  endif
   static double __test(double);
   static long double __test(long double);
 
@@ -89,6 +121,8 @@ public:
 template <class _A1, class _A2 = void, class _A3 = void>
 class __promote : public __promote_imp<_A1, _A2, _A3> {};
 
+#endif // !defined(_LIBCPP_CLANG_VER) || _LIBCPP_CLANG_VER >= 1700
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_PROMOTE_H
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 96a3e92faa81f21efa2547244d43f7d464eb5ced..4d47dc57ba6547a034d6b392314539d01c5b53d8 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -724,6 +724,10 @@ const time_zone* current_zone()
 const tzdb& reload_tzdb();                                                       // C++20
 string remote_version();                                                         // C++20
 
+// [time.zone.exception], exception classes
+class nonexistent_local_time;                                                    // C++20
+class ambiguous_local_time;                                                      // C++20
+
 // [time.zone.info], information classes
 struct sys_info {                                                                // C++20
   sys_seconds   begin;
@@ -763,6 +767,21 @@ class time_zone {
 
   template<class Duration>
   sys_info get_info(const sys_time<Duration>& st) const;
+
+  template<class Duration>
+  local_info get_info(const local_time<Duration>& tp) const;
+
+  template<class Duration>
+  sys_time<common_type_t<Duration, seconds>>
+    to_sys(const local_time<Duration>& tp) const;
+
+  template<class Duration>
+  sys_time<common_type_t<Duration, seconds>>
+    to_sys(const local_time<Duration>& tp, choose z) const;
+
+  template<class Duration>
+  local_time<common_type_t<Duration, seconds>>
+    to_local(const sys_time<Duration>& tp) const;
 };
 bool operator==(const time_zone& x, const time_zone& y) noexcept;                // C++20
 strong_ordering operator<=>(const time_zone& x, const time_zone& y) noexcept;    // C++20
@@ -912,6 +931,7 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 #if _LIBCPP_STD_VER >= 20
 #  include <__chrono/calendar.h>
 #  include <__chrono/day.h>
+#  include <__chrono/exception.h>
 #  include <__chrono/hh_mm_ss.h>
 #  include <__chrono/literals.h>
 #  include <__chrono/local_info.h>
diff --git a/libcxx/include/deque b/libcxx/include/deque
index 555761aae6afd2435f32f1955457e59e913299b8..8ffd8193b74e9db3524031832aff526edab27689 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -482,10 +482,10 @@ public:
                 "original allocator");
   static_assert(is_nothrow_default_constructible<allocator_type>::value ==
                     is_nothrow_default_constructible<__pointer_allocator>::value,
-                "rebinding an allocator should not change excpetion guarantees");
+                "rebinding an allocator should not change exception guarantees");
   static_assert(is_nothrow_move_constructible<allocator_type>::value ==
                     is_nothrow_move_constructible<typename __map::allocator_type>::value,
-                "rebinding an allocator should not change excpetion guarantees");
+                "rebinding an allocator should not change exception guarantees");
 
 private:
   struct __deque_block_range {
diff --git a/libcxx/include/memory b/libcxx/include/memory
index 30fadce760d75290b568e308fb17bf7bed3dc84f..a8c0264eb9eb78b447090602f042043b8da5fbba 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -830,34 +830,34 @@ public:
 };
 
 template<class T>
-    bool atomic_is_lock_free(const shared_ptr<T>* p);                                     // Deprecated in C++20, removed in C++26
+    bool atomic_is_lock_free(const shared_ptr<T>* p);
 template<class T>
-    shared_ptr<T> atomic_load(const shared_ptr<T>* p);                                    // Deprecated in C++20, removed in C++26
+    shared_ptr<T> atomic_load(const shared_ptr<T>* p);
 template<class T>
-    shared_ptr<T> atomic_load_explicit(const shared_ptr<T>* p, memory_order mo);          // Deprecated in C++20, removed in C++26
+    shared_ptr<T> atomic_load_explicit(const shared_ptr<T>* p, memory_order mo);
 template<class T>
-    void atomic_store(shared_ptr<T>* p, shared_ptr<T> r);                                 // Deprecated in C++20, removed in C++26
+    void atomic_store(shared_ptr<T>* p, shared_ptr<T> r);
 template<class T>
-    void atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);       // Deprecated in C++20, removed in C++26
+    void atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
 template<class T>
-    shared_ptr<T> atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r);                     // Deprecated in C++20, removed in C++26
+    shared_ptr<T> atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r);
 template<class T>
     shared_ptr<T>
-    atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);         // Deprecated in C++20, removed in C++26
+    atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
 template<class T>
     bool
-    atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);    // Deprecated in C++20, removed in C++26
+    atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
 template<class T>
     bool
-    atomic_compare_exchange_strong( shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w); // Deprecated in C++20, removed in C++26
+    atomic_compare_exchange_strong( shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
 template<class T>
     bool
-    atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,             // Deprecated in C++20, removed in C++26
+    atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
                                           shared_ptr<T> w, memory_order success,
                                           memory_order failure);
 template<class T>
     bool
-    atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,           // Deprecated in C++20, removed in C++26
+    atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
                                             shared_ptr<T> w, memory_order success,
                                             memory_order failure);
 // Hash support
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 48391b2a12095d41859f550e89397cf69bc6c7f4..e03ea989eb37191f23c9b3078c1bfa8aa30095c4 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1101,6 +1101,7 @@ module std_private_chrono_duration               [system] {
   header "__chrono/duration.h"
   export std_private_type_traits_is_convertible
 }
+module std_private_chrono_exception              [system] { header "__chrono/exception.h" }
 module std_private_chrono_file_clock             [system] { header "__chrono/file_clock.h" }
 module std_private_chrono_formatter              [system] {
   header "__chrono/formatter.h"
@@ -1113,7 +1114,10 @@ module std_private_chrono_high_resolution_clock  [system] {
 }
 module std_private_chrono_leap_second            [system] { header "__chrono/leap_second.h" }
 module std_private_chrono_literals               [system] { header "__chrono/literals.h" }
-module std_private_chrono_local_info             [system] { header "__chrono/local_info.h" }
+module std_private_chrono_local_info             [system] {
+  header "__chrono/local_info.h"
+  export std_private_chrono_sys_info
+}
 module std_private_chrono_month                  [system] { header "__chrono/month.h" }
 module std_private_chrono_month_weekday          [system] { header "__chrono/month_weekday.h" }
 module std_private_chrono_monthday               [system] { header "__chrono/monthday.h" }
diff --git a/libcxx/include/regex b/libcxx/include/regex
index b3869d36de1dfb4f5b80c2b9349b4847ab6e247d..46968357d39747d887c2e105dd579487ff84d864 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -4700,6 +4700,9 @@ private:
 
   template <class, class>
   friend class __lookahead;
+
+  template <class, class, class>
+  friend class regex_iterator;
 };
 
 template <class _BidirectionalIterator, class _Allocator>
@@ -5410,7 +5413,9 @@ template <class _BidirectionalIterator, class _CharT, class _Traits>
 regex_iterator<_BidirectionalIterator, _CharT, _Traits>&
 regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() {
   __flags_ |= regex_constants::__no_update_pos;
-  _BidirectionalIterator __start = __match_[0].second;
+  _BidirectionalIterator __start        = __match_[0].second;
+  _BidirectionalIterator __prefix_start = __start;
+
   if (__match_[0].first == __match_[0].second) {
     if (__start == __end_) {
       __match_ = value_type();
@@ -5424,9 +5429,21 @@ regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() {
     else
       ++__start;
   }
+
   __flags_ |= regex_constants::match_prev_avail;
-  if (!std::regex_search(__start, __end_, __match_, *__pregex_, __flags_))
+  if (!std::regex_search(__start, __end_, __match_, *__pregex_, __flags_)) {
     __match_ = value_type();
+
+  } else {
+    // The Standard mandates that if `regex_search` returns true ([re.regiter.incr]), "`match.prefix().first` shall be
+    // equal to the previous value of `match[0].second`... It is unspecified how the implementation makes these
+    // adjustments." The adjustment is necessary if we incremented `__start` above (the branch that deals with
+    // zero-length matches).
+    auto& __prefix = __match_.__prefix_;
+    __prefix.first = __prefix_start;
+    __prefix.matched = __prefix.first != __prefix.second;
+  }
+
   return *this;
 }
 
diff --git a/libcxx/modules/std/chrono.inc b/libcxx/modules/std/chrono.inc
index 813322a1797f6af83da703514bd6fd407600d551..f12c2425592e0bf9e88fc2ae4dce47feda70f3e4 100644
--- a/libcxx/modules/std/chrono.inc
+++ b/libcxx/modules/std/chrono.inc
@@ -209,13 +209,12 @@ export namespace std {
     using std::chrono::reload_tzdb;
     using std::chrono::remote_version;
 
-#    if 0
+#  endif //  !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
+         //  !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
     // [time.zone.exception], exception classes
     using std::chrono::ambiguous_local_time;
     using std::chrono::nonexistent_local_time;
-#    endif // if 0
-#  endif   //  !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
-           //  !defined(_LIBCPP_HAS_NO_LOCALIZATION)
 
     // [time.zone.info], information classes
     using std::chrono::local_info;
@@ -224,11 +223,8 @@ export namespace std {
 #  if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                            \
       !defined(_LIBCPP_HAS_NO_LOCALIZATION)
 
-#    if 0
     // [time.zone.timezone], class time_zone
     using std::chrono::choose;
-#    endif // if 0
-
     using std::chrono::time_zone;
 
 #    if 0
diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc
index b23c27707afd637024fd8b7f6a8c79363707e3e0..56c621c0cf17fb8dc7f3e3bf4df0ce781ca0e02a 100644
--- a/libcxx/modules/std/memory.inc
+++ b/libcxx/modules/std/memory.inc
@@ -190,9 +190,6 @@ export namespace std {
   //  using std::inout_ptr;
 
 #ifndef _LIBCPP_HAS_NO_THREADS
-
-#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
-
   // [depr.util.smartptr.shared.atomic]
   using std::atomic_is_lock_free;
 
@@ -209,8 +206,5 @@ export namespace std {
   using std::atomic_compare_exchange_strong_explicit;
   using std::atomic_compare_exchange_weak;
   using std::atomic_compare_exchange_weak_explicit;
-
-#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
-
 #endif // _LIBCPP_HAS_NO_THREADS
 } // namespace std
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 65e6ce2c4da43a7119c62fc4e3cb437da686607e..9e6c70335a794018fffb348531f130323e7f5fc6 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -340,6 +340,9 @@ if (LIBCXX_ENABLE_LOCALIZATION AND LIBCXX_ENABLE_FILESYSTEM AND LIBCXX_ENABLE_TI
     include/tzdb/types_private.h
     include/tzdb/tzdb_list_private.h
     include/tzdb/tzdb_private.h
+    # TODO TZDB The exception could be moved in chrono once the TZDB library
+    # is no longer experimental.
+    chrono_exception.cpp
     time_zone.cpp
     tzdb.cpp
     tzdb_list.cpp
diff --git a/llvm/lib/Target/AArch64/AArch64PassRegistry.def b/libcxx/src/chrono_exception.cpp
similarity index 50%
rename from llvm/lib/Target/AArch64/AArch64PassRegistry.def
rename to libcxx/src/chrono_exception.cpp
index ca944579f93a935f981d0b490a517cb561fad11c..bea2ad110310a306f45fb89a77302f3f321bb40b 100644
--- a/llvm/lib/Target/AArch64/AArch64PassRegistry.def
+++ b/libcxx/src/chrono_exception.cpp
@@ -1,20 +1,22 @@
-//===- AArch64PassRegistry.def - Registry of AArch64 passes -----*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// This file is used as the registry of passes that are part of the
-// AArch64 backend.
-//
-//===----------------------------------------------------------------------===//
 
-// NOTE: NO INCLUDE GUARD DESIRED!
+#include <chrono>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace chrono {
+
+_LIBCPP_AVAILABILITY_TZDB
+_LIBCPP_EXPORTED_FROM_ABI nonexistent_local_time::~nonexistent_local_time() = default; // key function
+_LIBCPP_AVAILABILITY_TZDB
+_LIBCPP_EXPORTED_FROM_ABI ambiguous_local_time::~ambiguous_local_time() = default; // key function
+
+} // namespace chrono
 
-#ifndef LOOP_PASS
-#define LOOP_PASS(NAME, CREATE_PASS)
-#endif
-LOOP_PASS("aarch64-lit", AArch64LoopIdiomTransformPass())
-#undef LOOP_PASS
+_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/time_zone.cpp b/libcxx/src/time_zone.cpp
index 928f3d2855e456f809e2e787d612f2c23d39fd49..764a89ab513c863f0b70a36bd561548535217ff8 100644
--- a/libcxx/src/time_zone.cpp
+++ b/libcxx/src/time_zone.cpp
@@ -34,6 +34,7 @@
 #include <chrono>
 #include <expected>
 #include <map>
+#include <numeric>
 #include <ranges>
 
 #include "include/tzdb/time_zone_private.h"
@@ -903,6 +904,152 @@ time_zone::__get_info(sys_seconds __time) const {
   std::__throw_runtime_error("tzdb: corrupt db");
 }
 
+// Is the "__local_time" present in "__first" and "__second". If so the
+// local_info has an ambiguous result.
+[[nodiscard]] static bool
+__is_ambiguous(local_seconds __local_time, const sys_info& __first, const sys_info& __second) {
+  std::chrono::local_seconds __end_first{__first.end.time_since_epoch() + __first.offset};
+  std::chrono::local_seconds __begin_second{__second.begin.time_since_epoch() + __second.offset};
+
+  return __local_time < __end_first && __local_time >= __begin_second;
+}
+
+// Determines the result of the "__local_time". This expects the object
+// "__first" to be earlier in time than "__second".
+[[nodiscard]] static local_info
+__get_info(local_seconds __local_time, const sys_info& __first, const sys_info& __second) {
+  std::chrono::local_seconds __end_first{__first.end.time_since_epoch() + __first.offset};
+  std::chrono::local_seconds __begin_second{__second.begin.time_since_epoch() + __second.offset};
+
+  if (__local_time < __end_first) {
+    if (__local_time >= __begin_second)
+      // |--------|
+      //        |------|
+      //         ^
+      return {local_info::ambiguous, __first, __second};
+
+    // |--------|
+    //          |------|
+    //         ^
+    return {local_info::unique, __first, sys_info{}};
+  }
+
+  if (__local_time < __begin_second)
+    // |--------|
+    //             |------|
+    //           ^
+    return {local_info::nonexistent, __first, __second};
+
+  // |--------|
+  //          |------|
+  //           ^
+  return {local_info::unique, __second, sys_info{}};
+}
+
+[[nodiscard]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_EXPORTED_FROM_ABI local_info
+time_zone::__get_info(local_seconds __local_time) const {
+  seconds __local_seconds = __local_time.time_since_epoch();
+
+  /* An example of a typical year with a DST switch displayed in local time.
+   *
+   * At the first of April the time goes forward one hour. This means the
+   * time marked with ~~ is not a valid local time. This is represented by the
+   * nonexistent value in local_info.result.
+   *
+   * At the first of November the time goes backward one hour. This means the
+   * time marked with ^^ happens twice. This is represented by the ambiguous
+   * value in local_info.result.
+   *
+   * 2020.11.01                  2021.04.01              2021.11.01
+   * offset +05                  offset +05              offset +05
+   * save    0s                  save    1h              save    0s
+   * |------------//----------|
+   *                             |---------//--------------|
+   *                                                    |-------------
+   *                           ~~                        ^^
+   *
+   * These shifts can happen due to changes in the current time zone for a
+   * location. For example, Indian/Kerguelen switched only once. In 1950 from an
+   * offset of 0 hours to an offset of +05 hours.
+   *
+   * During all these shifts the UTC time will not have gaps.
+   */
+
+  // The code needs to determine the system time for the local time. There is no
+  // information available. Assume the offset between system time and local time
+  // is 0s. This gives an initial estimate.
+  sys_seconds __guess{__local_seconds};
+  sys_info __info = __get_info(__guess);
+
+  // At this point the offset can be used to determine an estimate for the local
+  // time. Before doing that, determine the offset and validate whether the
+  // local time is the range [chrono::local_seconds::min(),
+  // chrono::local_seconds::max()).
+  if (__local_seconds < 0s && __info.offset > 0s)
+    if (__local_seconds - chrono::local_seconds::min().time_since_epoch() < __info.offset)
+      return {-1, __info, {}};
+
+  if (__local_seconds > 0s && __info.offset < 0s)
+    if (chrono::local_seconds::max().time_since_epoch() - __local_seconds < -__info.offset)
+      return {-2, __info, {}};
+
+  // Based on the information found in the sys_info, the local time can be
+  // converted to a system time. This resulting time can be in the following
+  // locations of the sys_info:
+  //
+  //                             |---------//--------------|
+  //                           1   2.1      2.2         2.3  3
+  //
+  // 1. The estimate is before the returned sys_info object.
+  //    The result is either non-existent or unique in the previous sys_info.
+  // 2. The estimate is in the sys_info object
+  //    - If the sys_info begin is not sys_seconds::min(), then it might be at
+  //      2.1 and could be ambiguous with the previous or unique.
+  //    - If sys_info end is not sys_seconds::max(), then it might be at 2.3
+  //      and could be ambiguous with the next or unique.
+  //    - Else it is at 2.2 and always unique. This case happens when a
+  //      time zone has no transitions. For example, UTC or GMT+1.
+  // 3. The estimate is after the returned sys_info object.
+  //    The result is either non-existent or unique in the next sys_info.
+  //
+  // There is no specification where the "middle" starts. Similar issues can
+  // happen when sys_info objects are "short", then "unique in the next" could
+  // become "ambiguous in the next and the one following". Theoretically there
+  // is the option of the following time-line
+  //
+  // |------------|
+  //           |----|
+  //       |-----------------|
+  //
+  // However the local_info object only has 2 sys_info objects, so this option
+  // is not tested.
+
+  sys_seconds __sys_time{__local_seconds - __info.offset};
+  if (__sys_time < __info.begin)
+    // Case 1 before __info
+    return chrono::__get_info(__local_time, __get_info(__info.begin - 1s), __info);
+
+  if (__sys_time >= __info.end)
+    // Case 3 after __info
+    return chrono::__get_info(__local_time, __info, __get_info(__info.end));
+
+  // Case 2 in __info
+  if (__info.begin != sys_seconds::min()) {
+    // Case 2.1 Not at the beginning, when not ambiguous the result should test
+    // case 2.3.
+    sys_info __prev = __get_info(__info.begin - 1s);
+    if (__is_ambiguous(__local_time, __prev, __info))
+      return {local_info::ambiguous, __prev, __info};
+  }
+
+  if (__info.end == sys_seconds::max())
+    // At the end so it's case 2.2
+    return {local_info::unique, __info, sys_info{}};
+
+  // This tests case 2.2 or case 2.3.
+  return chrono::__get_info(__local_time, __info, __get_info(__info.end));
+}
+
 } // namespace chrono
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 3ca01cad451a21b8512e19cb75ec4507eb008926..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// bool
-// atomic_compare_exchange_strong(shared_ptr<T>* p, shared_ptr<T>* v,
-//                                shared_ptr<T> w);                     // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> v(new int(3));
-  std::shared_ptr<int> w(new int(2));
-  // expected-warning@+1 {{'atomic_compare_exchange_strong<int>' is deprecated}}
-  std::ignore = std::atomic_compare_exchange_strong(&p, &v, w);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp
deleted file mode 100644
index 93384772c8afc7c7b281b58936e6523d6a656962..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// bool
-// atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
-//                                         shared_ptr<T> w, memory_order success,
-//                                         memory_order failure);                 // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> v(new int(3));
-  std::shared_ptr<int> w(new int(2));
-  // expected-warning@+2 {{'atomic_compare_exchange_strong_explicit<int>' is deprecated}}
-  std::ignore =
-      std::atomic_compare_exchange_strong_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 30ca70523fb647d4f7449a7452f6ed8f488c4ef8..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// bool
-// atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v,
-//                              shared_ptr<T> w);                    // Deprecated in C++20, removed in C++26
-
-#include <memory>
-#include <tuple>
-
-int main(int, char**) {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> v(new int(3));
-  std::shared_ptr<int> w(new int(2));
-  // expected-warning@+1 {{'atomic_compare_exchange_weak<int>' is deprecated}}
-  std::ignore = std::atomic_compare_exchange_weak(&p, &v, w);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp
deleted file mode 100644
index ae6e98d0692d24645cf54d60cb1426edbb678105..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// bool
-// atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
-//                                       shared_ptr<T> w, memory_order success,
-//                                       memory_order failure);                 // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> v(new int(3));
-  std::shared_ptr<int> w(new int(2));
-  // expected-warning@+2 {{'atomic_compare_exchange_weak_explicit<int>' is deprecated}}
-  std::ignore =
-      std::atomic_compare_exchange_weak_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 0e4bc1bc50b18b2fb50286ffd886e143ee97cf09..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// shared_ptr<T>
-// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
-
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> r(new int(3));
-  // expected-warning@+1 {{'atomic_exchange<int>' is deprecated}}
-  std::ignore = std::atomic_exchange(&p, r);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp
deleted file mode 100644
index 5a6a2bfe4c1e298206190e185dbb0097889f95bc..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// shared_ptr<T>
-// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)  // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(4));
-  std::shared_ptr<int> r(new int(3));
-  // expected-warning@+1 {{'atomic_exchange_explicit<int>' is deprecated}}
-  std::ignore = std::atomic_exchange_explicit(&p, r, std::memory_order_seq_cst);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 6799d31ce1fa951db92384f93f7111f6669fdf5d..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template<class T>
-// bool
-// atomic_is_lock_free(const shared_ptr<T>* p);    // Deprecated in C++20, removed in C++26
-
-#include <memory>
-#include <tuple>
-
-void test() {
-  const std::shared_ptr<int> p(new int(3));
-  // expected-warning@+1 {{'atomic_is_lock_free<int>' is deprecated}}
-  std::ignore = std::atomic_is_lock_free(&p);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 768ad0ded983e7c8754fcb1ae4282d3b795fb62e..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// shared_ptr<T>
-// atomic_load(const shared_ptr<T>* p)      // Deprecated in C++20, removed in C++26
-
-#include <memory>
-#include <tuple>
-
-void test() {
-  std::shared_ptr<int> p(new int(3));
-  // expected-warning@+1 {{'atomic_load<int>' is deprecated}}
-  std::ignore = std::atomic_load(&p);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp
deleted file mode 100644
index dc56193324f04f4b1d933ab02bd45215cb13fa39..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// shared_ptr<T>
-// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)     // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-#include <tuple>
-
-void test() {
-  const std::shared_ptr<int> p(new int(3));
-  // expected-warning@+1 {{'atomic_load_explicit<int>' is deprecated}}
-  std::ignore = std::atomic_load_explicit(&p, std::memory_order_relaxed);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 38b8df6f24393c81ddf4ace9ecd994f92628732a..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// void
-// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
-
-#include <memory>
-
-void test() {
-  std::shared_ptr<int> p;
-  std::shared_ptr<int> r(new int(3));
-  // expected-warning@+1 {{'atomic_store<int>' is deprecated}}
-  std::atomic_store(&p, r);
-}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp
deleted file mode 100644
index 18f59c461869969d1a5061c20f43bf26e1394e94..0000000000000000000000000000000000000000
--- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <memory>
-
-// shared_ptr
-
-// template <class T>
-// void
-// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)  // Deprecated in C++20, removed in C++26
-
-#include <atomic>
-#include <memory>
-
-void test() {
-  std::shared_ptr<int> p;
-  std::shared_ptr<int> r(new int(3));
-  // expected-warning@+1 {{'atomic_store_explicit<int>' is deprecated}}
-  std::atomic_store_explicit(&p, r, std::memory_order_seq_cst);
-}
diff --git a/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp b/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
index a5ce5d16581306c4c28bc427feb05a60b1b41b20..cba7916ff2c646f1055b913a82acdc84486ee4b6 100644
--- a/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/diagnostics/chrono.nodiscard.verify.cpp
@@ -48,8 +48,13 @@ void test() {
 
   {
     std::chrono::sys_seconds s{};
+    std::chrono::local_seconds l{};
+    std::chrono::choose z = std::chrono::choose::earliest;
     tz.name();           // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
     tz.get_info(s);      // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+    tz.get_info(l);      // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+    tz.to_sys(l);        // not nodiscard
+    tz.to_sys(l, z);     // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
     operator==(tz, tz);  // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
     operator<=>(tz, tz); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   }
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..73e6bf2846f0e0dba5ee239a2d01a0550caccf77
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// REQUIRES: has-unix-headers
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// <chrono>
+
+//  class ambiguous_local_time
+//
+//  template<class Duration>
+//    ambiguous_local_time(const local_time<Duration>& tp, const local_info& i);
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// [time.zone.exception.ambig]/2
+//   Preconditions: i.result == local_info::ambiguous is true.
+int main(int, char**) {
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::ambiguous_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{-1, //  this is not one of the "named" result values
+                                  std::chrono::sys_info{},
+                                  std::chrono::sys_info{}}}),
+      "creating an ambiguous_local_time from a local_info that is not ambiguous");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::ambiguous_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{std::chrono::local_info::unique, std::chrono::sys_info{}, std::chrono::sys_info{}}}),
+      "creating an ambiguous_local_time from a local_info that is not ambiguous");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::ambiguous_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{
+              std::chrono::local_info::nonexistent, std::chrono::sys_info{}, std::chrono::sys_info{}}}),
+      "creating an ambiguous_local_time from a local_info that is not ambiguous");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fdd9f79958f9805bf09e053c4b7928ea1bce89e9
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// REQUIRES: has-unix-headers
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// <chrono>
+
+//  class nonexistent_local_time
+//
+//  template<class Duration>
+//    nonexistent_local_time(const local_time<Duration>& tp, const local_info& i);
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// [time.zone.exception.nonexist]/2
+//   Preconditions: i.result == local_info::nonexistent is true.
+int main(int, char**) {
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::nonexistent_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{-1, //  this is not one of the "named" result values
+                                  std::chrono::sys_info{},
+                                  std::chrono::sys_info{}}}),
+      "creating an nonexistent_local_time from a local_info that is not non-existent");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::nonexistent_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{std::chrono::local_info::unique, std::chrono::sys_info{}, std::chrono::sys_info{}}}),
+      "creating an nonexistent_local_time from a local_info that is not non-existent");
+
+  TEST_LIBCPP_ASSERT_FAILURE(
+      (std::chrono::nonexistent_local_time{
+          std::chrono::local_seconds{},
+          std::chrono::local_info{
+              std::chrono::local_info::ambiguous, std::chrono::sys_info{}, std::chrono::sys_info{}}}),
+      "creating an nonexistent_local_time from a local_info that is not non-existent");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/choose.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/choose.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..23ef9c848096083cec4e9fa6f90b2990a826ca04
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/choose.pass.cpp
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// enum class choose;
+
+#include <chrono>
+#include <type_traits>
+#include <cassert>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  using E = std::chrono::choose;
+  static_assert(std::is_enum_v<E>);
+
+  // Check that E is a scoped enum by checking for conversions.
+  using UT = std::underlying_type_t<E>;
+  static_assert(!std::is_convertible_v<E, UT>);
+
+  [[maybe_unused]] const E& early = E::earliest;
+  [[maybe_unused]] const E& late  = E::latest;
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d9ca1c80751cce7bd311ce3557243f360af1295d
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// REQUIRES: has-unix-headers
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// template <class _Duration>
+// local_time<common_type_t<Duration, seconds>>
+//   to_local(const sys_time<Duration>& tp) const;
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// Tests values that cannot be converted. To make sure the test is does not depend on changes
+// in the database it uses a time zone with a fixed offset.
+int main(int, char**) {
+  TEST_LIBCPP_ASSERT_FAILURE(std::chrono::locate_zone("Etc/GMT+1")->to_local(std::chrono::sys_seconds::min()),
+                             "cannot convert the system time; it would be before the minimum local clock value");
+
+  // TODO TZDB look why std::chrono::sys_seconds::max()  fails
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::locate_zone("Etc/GMT-1")->to_local(std::chrono::sys_seconds::max() - std::chrono::seconds(1)),
+      "cannot convert the system time; it would be after the maximum local clock value");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3a2ff00088676b315a35f9bdcef453d3a7af6f45
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// REQUIRES: has-unix-headers
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// <chrono>
+
+// template <class _Duration>
+//   sys_time<common_type_t<Duration, seconds>>
+//     to_sys(const local_time<Duration>& tp) const;
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// Tests values that cannot be converted. To make sure the test is does not depend on changes
+// in the database it uses a time zone with a fixed offset.
+int main(int, char**) {
+  TEST_LIBCPP_ASSERT_FAILURE(std::chrono::locate_zone("Etc/GMT-1")->to_sys(std::chrono::local_seconds::min()),
+                             "cannot convert the local time; it would be before the minimum system clock value");
+
+  // TODO TZDB look why std::chrono::local_seconds::max()  fails
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::locate_zone("Etc/GMT+1")->to_sys(std::chrono::local_seconds::max() - std::chrono::seconds(1)),
+      "cannot convert the local time; it would be after the maximum system clock value");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..65429345ae794f70b878366099130855a4f67fcc
--- /dev/null
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// REQUIRES: has-unix-headers
+// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+
+// <chrono>
+
+// template <class _Duration>
+//   sys_time<common_type_t<Duration, seconds>>
+//     to_sys(const local_time<Duration>& tp, choose z) const;
+
+#include <chrono>
+
+#include "check_assertion.h"
+
+// Tests values that cannot be converted. To make sure the test is does not depend on changes
+// in the database it uses a time zone with a fixed offset.
+int main(int, char**) {
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::locate_zone("Etc/GMT-1")->to_sys(std::chrono::local_seconds::min(), std::chrono::choose::earliest),
+      "cannot convert the local time; it would be before the minimum system clock value");
+
+  // TODO TZDB look why std::chrono::local_seconds::max()  fails
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::chrono::locate_zone("Etc/GMT+1")
+          ->to_sys(std::chrono::local_seconds::max() - std::chrono::seconds(1), std::chrono::choose::latest),
+      "cannot convert the local time; it would be after the maximum system clock value");
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index 92601fab5b7730cf6d5fddfe584cc60cd8427571..b0431d9b2b551afeb5e8bba011b2490390add1a1 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -187,9 +187,6 @@ condition_variable type_traits
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
-coroutine cstdint
-coroutine cstring
 coroutine iosfwd
 coroutine limits
 coroutine type_traits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index c05eb42deb9a1fc61dbbd23c1d7e7b37ae9af620..6fc8fe5858c0dcb6aeb9866dd4318c8b77504353 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -188,9 +188,6 @@ condition_variable type_traits
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
-coroutine cstdint
-coroutine cstring
 coroutine iosfwd
 coroutine limits
 coroutine type_traits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 09252b7b7d2dba828e78b2c733ce8b1ec22cab59..5771e2ba0761cd8ddb2e0813100922fdc5d4ab39 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -189,9 +189,6 @@ condition_variable type_traits
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
-coroutine cstdint
-coroutine cstring
 coroutine iosfwd
 coroutine limits
 coroutine type_traits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 09252b7b7d2dba828e78b2c733ce8b1ec22cab59..5771e2ba0761cd8ddb2e0813100922fdc5d4ab39 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -189,9 +189,6 @@ condition_variable type_traits
 condition_variable typeinfo
 condition_variable version
 coroutine compare
-coroutine cstddef
-coroutine cstdint
-coroutine cstring
 coroutine iosfwd
 coroutine limits
 coroutine type_traits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index ce4ccc3d1161537f5e16cc3a43ffb8ee1e246ad6..406665a21353678622003800291dd421c0729805 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -111,26 +111,19 @@ charconv limits
 charconv new
 charconv type_traits
 charconv version
-chrono array
 chrono bit
-chrono cctype
-chrono cerrno
 chrono charconv
-chrono clocale
 chrono cmath
 chrono compare
 chrono concepts
 chrono cstddef
 chrono cstdint
-chrono cstdlib
 chrono cstring
 chrono ctime
-chrono cwchar
+chrono format
 chrono forward_list
 chrono limits
 chrono locale
-chrono new
-chrono optional
 chrono ostream
 chrono ratio
 chrono sstream
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index 62d931c0eebadef2cedf8ffabdfb0233c718e55b..05c7a6ea18d796ecda26746025b554dcb712c6fb 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -67,30 +67,20 @@ charconv initializer_list
 charconv limits
 charconv new
 charconv version
-chrono array
-chrono cctype
-chrono cerrno
-chrono clocale
 chrono cmath
 chrono compare
 chrono cstddef
 chrono cstdint
-chrono cstdlib
-chrono cstring
 chrono ctime
-chrono cwchar
+chrono format
 chrono forward_list
 chrono limits
-chrono new
-chrono optional
 chrono ostream
 chrono ratio
 chrono sstream
 chrono stdexcept
 chrono string
 chrono string_view
-chrono tuple
-chrono typeinfo
 chrono vector
 chrono version
 cinttypes cstdint
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index f68249aeec78c940652e30c6bf3662cd0443e8e9..05c7a6ea18d796ecda26746025b554dcb712c6fb 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -67,30 +67,20 @@ charconv initializer_list
 charconv limits
 charconv new
 charconv version
-chrono array
-chrono cctype
-chrono cerrno
-chrono clocale
 chrono cmath
 chrono compare
 chrono cstddef
 chrono cstdint
-chrono cstdlib
-chrono cstring
 chrono ctime
-chrono cwchar
+chrono format
 chrono forward_list
 chrono limits
-chrono new
-chrono optional
 chrono ostream
 chrono ratio
 chrono sstream
 chrono stdexcept
 chrono string
 chrono string_view
-chrono tuple
-chrono typeinfo
 chrono vector
 chrono version
 cinttypes cstdint
@@ -176,29 +166,6 @@ experimental/simd limits
 experimental/type_traits initializer_list
 experimental/type_traits type_traits
 experimental/utility utility
-experimental/vector experimental/memory_resource
-experimental/vector vector
-ext/hash_map algorithm
-ext/hash_map cmath
-ext/hash_map cstddef
-ext/hash_map cstdint
-ext/hash_map cstring
-ext/hash_map functional
-ext/hash_map initializer_list
-ext/hash_map limits
-ext/hash_map new
-ext/hash_map stdexcept
-ext/hash_map string
-ext/hash_set algorithm
-ext/hash_set cmath
-ext/hash_set cstddef
-ext/hash_set cstdint
-ext/hash_set cstring
-ext/hash_set functional
-ext/hash_set initializer_list
-ext/hash_set limits
-ext/hash_set new
-ext/hash_set string
 filesystem compare
 filesystem cstddef
 filesystem cstdint
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
index 7350c1ddf0e90120466abbed53377bee6a3db1f9..40a475ec38b72302077f9ab80614b0fc01e651f0 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp
@@ -6,9 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: LIBCXX-AIX-FIXME
 // XFAIL: !has-64-bit-atomics
 
+// Older versions of clang have a bug with atomic builtins affecting double and long double.
+// Fixed by 5fdd0948.
+// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+
 // https://github.com/llvm/llvm-project/issues/72893
 // XFAIL: target={{x86_64-.*}} && tsan
 
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
index 84dcde5f2784f2eefc43fe334ce7f43adac5cf36..9e798a2a519ff03086b4efe37a75ce63183e0983 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp
@@ -6,9 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: LIBCXX-AIX-FIXME
 // XFAIL: !has-64-bit-atomics
 
+// Older versions of clang have a bug with atomic builtins affecting double and long double.
+// Fixed by 5fdd0948.
+// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+
 // https://github.com/llvm/llvm-project/issues/72893
 // XFAIL: target={{x86_64-.*}} && tsan
 
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
index 386a393e3550397e1b60ef2dcb29a505bf315c65..732bd4d7e5dc2bc16d925b2a0a1d9d6d5941adab 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp
@@ -6,9 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: LIBCXX-AIX-FIXME
 // XFAIL: !has-64-bit-atomics
 
+// Older versions of clang have a bug with atomic builtins affecting double and long double.
+// Fixed by 5fdd0948.
+// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+
 // floating-point-type operator-=(floating-point-type) volatile noexcept;
 // floating-point-type operator-=(floating-point-type) noexcept;
 
diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
index afd06d537c7adbe8c4160afc3ce4875aab6cd117..1821aca42c7988eb7e14b28e129e55f4e96626a8 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp
@@ -6,9 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: LIBCXX-AIX-FIXME
 // XFAIL: !has-64-bit-atomics
 
+// Older versions of clang have a bug with atomic builtins affecting double and long double.
+// Fixed by 5fdd0948.
+// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18)
+
 // floating-point-type operator+=(floating-point-type) volatile noexcept;
 // floating-point-type operator+=(floating-point-type) noexcept;
 
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
index 8fcc811f6df3e75b4589b649738fd2093f145110..7d91ca0eada1d36fabd03bbe56062548b7d2445f 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp
@@ -8,9 +8,9 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
-// FIXME: Fatal error with following targets (remove XFAIL when fixed):
+// Older versions of clang may encounter a backend error (see 0295c2ad):
 //   Pass-by-value arguments with alignment greater than register width are not supported.
-// XFAIL: target=powerpc{{.*}}-ibm-aix7.2.5.7
+// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && (clang-17 || clang-18)
 
 // <experimental/simd>
 //
diff --git a/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp b/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2f1b6280deb780a36c4491511a388511f93882b2
--- /dev/null
+++ b/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <regex>
+
+// Test that replacing zero-length matches works correctly.
+
+#include <cassert>
+#include <regex>
+#include <string>
+#include "test_macros.h"
+
+int main(int, char**) {
+  // Various patterns that produce zero-length matches.
+  assert(std::regex_replace("abc", std::regex(""), "!") == "!a!b!c!");
+  assert(std::regex_replace("abc", std::regex("X*"), "!") == "!a!b!c!");
+  assert(std::regex_replace("abc", std::regex("X{0,3}"), "!") == "!a!b!c!");
+
+  // Replacement string has several characters.
+  assert(std::regex_replace("abc", std::regex(""), "[!]") == "[!]a[!]b[!]c[!]");
+
+  // Empty replacement string.
+  assert(std::regex_replace("abc", std::regex(""), "") == "abc");
+
+  // Empty input.
+  assert(std::regex_replace("", std::regex(""), "!") == "!");
+
+  // Not all matches are zero-length.
+  assert(std::regex_replace("abCabCa", std::regex("C*"), "!") == "!a!b!!a!b!!a!");
+
+  return 0;
+}
diff --git a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp
index 9332158f0de95f17fe3c78b4246269d3abc3cfb0..aefe86f1529809dcd96e01e8404053864669a8b8 100644
--- a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp
+++ b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp
@@ -17,102 +17,151 @@
 #include <iterator>
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        std::regex phone_numbers("\\d{3}-\\d{4}");
-        const char phone_book[] = "555-1234, 555-2345, 555-3456";
-        std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers);
-        std::cregex_iterator i2 = i;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 0);
-        assert((*i).str() == "555-1234");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        i++;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 10);
-        assert((*i).str() == "555-2345");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        i++;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 20);
-        assert((*i).str() == "555-3456");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        i++;
-        assert(i == std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-    }
-    {
-        std::regex phone_numbers("\\d{3}-\\d{4}");
-        const char phone_book[] = "555-1234, 555-2345, 555-3456";
-        std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers);
-        std::cregex_iterator i2 = i;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 0);
-        assert((*i).str() == "555-1234");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        ++i;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 10);
-        assert((*i).str() == "555-2345");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        ++i;
-        assert(i != std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i).size() == 1);
-        assert((*i).position() == 20);
-        assert((*i).str() == "555-3456");
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-        ++i;
-        assert(i == std::cregex_iterator());
-        assert(i2!= std::cregex_iterator());
-        assert((*i2).size() == 1);
-        assert((*i2).position() == 0);
-        assert((*i2).str() == "555-1234");
-    }
-    { // https://llvm.org/PR33681
-        std::regex rex(".*");
-        const char foo[] = "foo";
+void validate_prefixes(const std::regex& empty_matching_pattern) {
+  const char source[] = "abc";
+
+  std::cregex_iterator i(source, source + 3, empty_matching_pattern);
+  assert(!i->prefix().matched);
+  assert(i->prefix().length() == 0);
+  assert(i->prefix().first == source);
+  assert(i->prefix().second == source);
+
+  ++i;
+  assert(i->prefix().matched);
+  assert(i->prefix().length() == 1);
+  assert(i->prefix().first == source);
+  assert(i->prefix().second == source + 1);
+  assert(i->prefix().str() == "a");
+
+  ++i;
+  assert(i->prefix().matched);
+  assert(i->prefix().length() == 1);
+  assert(i->prefix().first == source + 1);
+  assert(i->prefix().second == source + 2);
+  assert(i->prefix().str() == "b");
+
+  ++i;
+  assert(i->prefix().matched);
+  assert(i->prefix().length() == 1);
+  assert(i->prefix().first == source + 2);
+  assert(i->prefix().second == source + 3);
+  assert(i->prefix().str() == "c");
+
+  ++i;
+  assert(i == std::cregex_iterator());
+}
+
+void test_prefix_adjustment() {
+  // Check that we correctly adjust the match prefix when dealing with zero-length matches -- this is explicitly
+  // required by the Standard ([re.regiter.incr]: "In all cases in which the call to `regex_search` returns true,
+  // `match.prefix().first` shall be equal to the previous value of `match[0].second`"). For a pattern that matches
+  // empty sequences, there is an implicit zero-length match between every character in a string -- make sure the
+  // prefix of each of these matches (except the first one) is the preceding character.
+
+  // An empty pattern produces zero-length matches.
+  validate_prefixes(std::regex(""));
+  // Any character repeated zero or more times can produce zero-length matches.
+  validate_prefixes(std::regex("X*"));
+  validate_prefixes(std::regex("X{0,3}"));
+}
+
+int main(int, char**) {
+  {
+    std::regex phone_numbers("\\d{3}-\\d{4}");
+    const char phone_book[] = "555-1234, 555-2345, 555-3456";
+    std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers);
+    std::cregex_iterator i2 = i;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 0);
+    assert((*i).str() == "555-1234");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    i++;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 10);
+    assert((*i).str() == "555-2345");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    i++;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 20);
+    assert((*i).str() == "555-3456");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    i++;
+    assert(i == std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+  }
+  {
+    std::regex phone_numbers("\\d{3}-\\d{4}");
+    const char phone_book[] = "555-1234, 555-2345, 555-3456";
+    std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers);
+    std::cregex_iterator i2 = i;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 0);
+    assert((*i).str() == "555-1234");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    ++i;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 10);
+    assert((*i).str() == "555-2345");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    ++i;
+    assert(i != std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i).size() == 1);
+    assert((*i).position() == 20);
+    assert((*i).str() == "555-3456");
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+    ++i;
+    assert(i == std::cregex_iterator());
+    assert(i2 != std::cregex_iterator());
+    assert((*i2).size() == 1);
+    assert((*i2).position() == 0);
+    assert((*i2).str() == "555-1234");
+  }
+  { // https://llvm.org/PR33681
+    std::regex rex(".*");
+    const char foo[] = "foo";
     //  The -1 is because we don't want the implicit null from the array.
-        std::cregex_iterator i(std::begin(foo), std::end(foo) - 1, rex);
-        std::cregex_iterator e;
-        assert(i != e);
-        assert((*i).size() == 1);
-        assert((*i).str() == "foo");
-
-        ++i;
-        assert(i != e);
-        assert((*i).size() == 1);
-        assert((*i).str() == "");
-
-        ++i;
-        assert(i == e);
-    }
+    std::cregex_iterator i(std::begin(foo), std::end(foo) - 1, rex);
+    std::cregex_iterator e;
+    assert(i != e);
+    assert((*i).size() == 1);
+    assert((*i).str() == "foo");
+
+    ++i;
+    assert(i != e);
+    assert((*i).size() == 1);
+    assert((*i).str() == "");
+
+    ++i;
+    assert(i == e);
+  }
+
+  test_prefix_adjustment();
 
   return 0;
 }
diff --git a/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/ctor.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/ctor.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a0b141631f83142eb7a7adaf7f8473cbf8f1b34e
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/ctor.pass.cpp
@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+//  class ambiguous_local_time
+//
+//  template<class Duration>
+//    ambiguous_local_time(const local_time<Duration>& tp, const local_info& i);
+
+#include <chrono>
+#include <string_view>
+
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+template <class Duration>
+static void
+test(const std::chrono::local_time<Duration>& tp, const std::chrono::local_info& i, std::string_view expected) {
+  std::chrono::ambiguous_local_time exception{tp, i};
+  std::string_view result = exception.what();
+  TEST_REQUIRE(result == expected,
+               TEST_WRITE_CONCATENATED("Expected output\n", expected, "\n\nActual output\n", result, '\n'));
+}
+
+// The constructor constructs the runtime_error base class with a specific
+// message. This implicitly tests what() too, since that is inherited from
+// runtime_error there is no separate test for what().
+int main(int, char**) {
+  using namespace std::literals::chrono_literals;
+
+  // There is no requirement on the ordering of PREV and NEXT so an "invalid"
+  // overlap is allowed. All tests with negative dates use the same order as
+  // positive tests.
+
+  test(std::chrono::local_time<std::chrono::nanoseconds>{-1ns},
+       std::chrono::local_info{
+           std::chrono::local_info::ambiguous,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "PREV"},
+
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "NEXT"}
+
+       },
+       R"(1969-12-31 23:59:59.999999999 is ambiguous.  It could be
+1969-12-31 23:59:59.999999999 PREV == 1969-12-31 22:59:59.999999999 UTC or
+1969-12-31 23:59:59.999999999 NEXT == 1969-12-31 23:59:59.999999999 UTC)");
+
+  test(std::chrono::local_time<std::chrono::microseconds>{0us},
+       std::chrono::local_info{
+           std::chrono::local_info::ambiguous,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "PREV"},
+
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "NEXT"}},
+       R"(1970-01-01 00:00:00.000000 is ambiguous.  It could be
+1970-01-01 00:00:00.000000 PREV == 1969-12-31 23:00:00.000000 UTC or
+1970-01-01 00:00:00.000000 NEXT == 1970-01-01 00:00:00.000000 UTC)");
+
+  test(std::chrono::local_time<std::chrono::milliseconds>{1ms},
+       std::chrono::local_info{
+           std::chrono::local_info::ambiguous,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "PREV"},
+
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "NEXT"}},
+       R"(1970-01-01 00:00:00.001 is ambiguous.  It could be
+1970-01-01 00:00:00.001 PREV == 1969-12-31 23:00:00.001 UTC or
+1970-01-01 00:00:00.001 NEXT == 1970-01-01 00:00:00.001 UTC)");
+
+  test(std::chrono::local_seconds{(std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch()},
+       std::chrono::local_info{
+           std::chrono::local_info::ambiguous,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / -21969},
+               std::chrono::sys_days{std::chrono::December / 31 / -21969},
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / -21970},
+               std::chrono::sys_days{std::chrono::March / 1 / -21970} + 23h,
+               1h,
+               60min,
+               "NEXT"}},
+       R"(-21970-01-01 00:00:00 is ambiguous.  It could be
+-21970-01-01 00:00:00 PREV == -21970-01-01 00:00:00 UTC or
+-21970-01-01 00:00:00 NEXT == -21971-12-31 23:00:00 UTC)");
+
+  test(
+      std::chrono::local_time<std::chrono::days>{
+          (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch()},
+      std::chrono::local_info{
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info{
+              std::chrono::sys_days{std::chrono::September / 1 / 21969},
+              std::chrono::sys_days{std::chrono::December / 31 / 21969},
+              0s,
+              0min,
+              "PREV"},
+          std::chrono::sys_info{
+              std::chrono::sys_days{std::chrono::January / 1 / 21970},
+              std::chrono::sys_days{std::chrono::March / 1 / 21970} + 23h,
+              1h,
+              60min,
+              "NEXT"}},
+      R"(21970-01-01 is ambiguous.  It could be
+21970-01-01 PREV == 21970-01-01 00:00:00 UTC or
+21970-01-01 NEXT == 21969-12-31 23:00:00 UTC)");
+
+  test(std::chrono::local_time<std::chrono::weeks>{},
+       std::chrono::local_info{
+           std::chrono::local_info::ambiguous,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969},
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970} + 23h,
+               1h,
+               60min,
+               "NEXT"}},
+       R"(1970-01-01 is ambiguous.  It could be
+1970-01-01 PREV == 1970-01-01 00:00:00 UTC or
+1970-01-01 NEXT == 1969-12-31 23:00:00 UTC)");
+
+  // Note months and years can not be streamed.
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/types.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/types.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..be54aedbbba61ea20841d3e14f8010821e7461f5
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.ambig/types.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// ADDITIONAL_COMPILE_FLAGS(clang): -Wno-deprecated
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-deprecated-copy-dtor
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class ambiguous_local_time : public runtime_error {
+// public:
+//   template<class Duration>
+//     ambiguous_local_time(const local_time<Duration>& tp, const local_info& i);
+// };
+
+#include <chrono>
+#include <stdexcept>
+#include <type_traits>
+
+// Basic properties
+static_assert(std::is_base_of_v<std::runtime_error, std::chrono::ambiguous_local_time>);
+static_assert(!std::is_default_constructible_v<std::chrono::ambiguous_local_time>);
+static_assert(std::is_destructible_v<std::chrono::ambiguous_local_time>);
+static_assert(std::is_copy_constructible_v<std::chrono::ambiguous_local_time>);
+static_assert(std::is_move_constructible_v<std::chrono::ambiguous_local_time>);
+static_assert(std::is_copy_assignable_v<std::chrono::ambiguous_local_time>);
+static_assert(std::is_move_assignable_v<std::chrono::ambiguous_local_time>);
+
+int main(int, char**) {
+  std::chrono::ambiguous_local_time e{
+      std::chrono::local_seconds{}, std::chrono::local_info{std::chrono::local_info::ambiguous, {}, {}}};
+
+  std::chrono::ambiguous_local_time copy = e;
+  copy                                   = e;
+
+  std::chrono::ambiguous_local_time move = std::move(e);
+  e                                      = move;
+  move                                   = std::move(e);
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/ctor.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/ctor.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ca03d832a7a9f504d0cfec469947912c3f71f6ad
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/ctor.pass.cpp
@@ -0,0 +1,172 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class nonexistent_local_time
+//
+// template<class Duration>
+// nonexistent_local_time(const local_time<Duration>& tp, const local_info& i);
+
+#include <chrono>
+#include <string_view>
+
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+template <class Duration>
+static void
+test(const std::chrono::local_time<Duration>& tp, const std::chrono::local_info& i, std::string_view expected) {
+  std::chrono::nonexistent_local_time exception{tp, i};
+  std::string_view result = exception.what();
+  TEST_REQUIRE(result == expected,
+               TEST_WRITE_CONCATENATED("Expected output\n", expected, "\n\nActual output\n", result, '\n'));
+}
+
+// The constructor constructs the runtime_error base class with a specific
+// message. This implicitly tests what() too, since that is inherited from
+// runtime_error there is no separate test for what().
+int main(int, char**) {
+  using namespace std::literals::chrono_literals;
+
+  // There is no requirement on the ordering of PREV and NEXT so an "invalid"
+  // gap is allowed. All tests with negative dates use the same order as
+  // positive tests.
+
+  test(std::chrono::local_time<std::chrono::nanoseconds>{-1ns},
+       std::chrono::local_info{
+           std::chrono::local_info::nonexistent,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "NEXT"}},
+       R"(1969-12-31 23:59:59.999999999 is in a gap between
+1969-12-31 23:00:00 PREV and
+1970-01-01 01:00:00 NEXT which are both equivalent to
+1969-12-31 23:00:00 UTC)");
+
+  test(std::chrono::local_time<std::chrono::microseconds>{0us},
+       std::chrono::local_info{
+           std::chrono::local_info::nonexistent,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "NEXT"}},
+       R"(1970-01-01 00:00:00.000000 is in a gap between
+1969-12-31 23:00:00 PREV and
+1970-01-01 01:00:00 NEXT which are both equivalent to
+1969-12-31 23:00:00 UTC)");
+
+  test(std::chrono::local_time<std::chrono::milliseconds>{1ms},
+       std::chrono::local_info{
+           std::chrono::local_info::nonexistent,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "NEXT"}},
+       R"(1970-01-01 00:00:00.001 is in a gap between
+1969-12-31 23:00:00 PREV and
+1970-01-01 01:00:00 NEXT which are both equivalent to
+1969-12-31 23:00:00 UTC)");
+
+  test(std::chrono::local_seconds{(std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch()},
+       std::chrono::local_info{
+           std::chrono::local_info::nonexistent,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / -21969},
+               std::chrono::sys_days{std::chrono::December / 31 / -21969} + 23h,
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / -21970},
+               std::chrono::sys_days{std::chrono::March / 1 / -21970},
+               1h,
+               60min,
+               "NEXT"}},
+       R"(-21970-01-01 00:00:00 is in a gap between
+-21969-12-31 23:00:00 PREV and
+-21970-01-01 01:00:00 NEXT which are both equivalent to
+-21969-12-31 23:00:00 UTC)");
+
+  test(
+      std::chrono::local_time<std::chrono::days>{
+          (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch()},
+      std::chrono::local_info{
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info{
+              std::chrono::sys_days{std::chrono::September / 1 / 21969},
+              std::chrono::sys_days{std::chrono::December / 31 / 21969} + 23h,
+              0s,
+              0min,
+              "PREV"},
+          std::chrono::sys_info{
+              std::chrono::sys_days{std::chrono::January / 1 / 21970},
+              std::chrono::sys_days{std::chrono::March / 1 / 21970},
+              1h,
+              60min,
+              "NEXT"}},
+      R"(21970-01-01 is in a gap between
+21969-12-31 23:00:00 PREV and
+21970-01-01 01:00:00 NEXT which are both equivalent to
+21969-12-31 23:00:00 UTC)");
+
+  test(std::chrono::local_time<std::chrono::weeks>{},
+       std::chrono::local_info{
+           std::chrono::local_info::nonexistent,
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::September / 1 / 1969},
+               std::chrono::sys_days{std::chrono::December / 31 / 1969} + 23h,
+               0s,
+               0min,
+               "PREV"},
+           std::chrono::sys_info{
+               std::chrono::sys_days{std::chrono::January / 1 / 1970},
+               std::chrono::sys_days{std::chrono::March / 1 / 1970},
+               1h,
+               60min,
+               "NEXT"}},
+       R"(1970-01-01 is in a gap between
+1969-12-31 23:00:00 PREV and
+1970-01-01 01:00:00 NEXT which are both equivalent to
+1969-12-31 23:00:00 UTC)");
+
+  // Note months and years can not be streamed.
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/types.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/types.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..85ebfab679e02d103ef45b2ae8255e08e0ce5e92
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.exception/time.zone.exception.nonexist/types.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// ADDITIONAL_COMPILE_FLAGS(clang): -Wno-deprecated
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-deprecated-copy-dtor
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class nonexistent_local_time : public runtime_error {
+// public:
+//   template<class Duration>
+//     nonexistent_local_time(const local_time<Duration>& tp, const local_info& i);
+// };
+
+#include <chrono>
+#include <stdexcept>
+#include <type_traits>
+
+// Basic properties
+static_assert(std::is_base_of_v<std::runtime_error, std::chrono::nonexistent_local_time>);
+static_assert(!std::is_default_constructible_v<std::chrono::nonexistent_local_time>);
+static_assert(std::is_destructible_v<std::chrono::nonexistent_local_time>);
+static_assert(std::is_copy_constructible_v<std::chrono::nonexistent_local_time>);
+static_assert(std::is_move_constructible_v<std::chrono::nonexistent_local_time>);
+static_assert(std::is_copy_assignable_v<std::chrono::nonexistent_local_time>);
+static_assert(std::is_move_assignable_v<std::chrono::nonexistent_local_time>);
+
+int main(int, char**) {
+  std::chrono::nonexistent_local_time e{
+      std::chrono::local_seconds{}, std::chrono::local_info{std::chrono::local_info::nonexistent, {}, {}}};
+
+  std::chrono::nonexistent_local_time copy = e;
+  copy                                     = e;
+
+  std::chrono::nonexistent_local_time move = std::move(e);
+  e                                        = move;
+  move                                     = std::move(e);
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6dc15974c44843dad1ced250908e92b8ece26be3
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp
@@ -0,0 +1,1302 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class time_zone;
+
+// template <class _Duration>
+//   local_info get_info(const local_time<_Duration>& time) const;
+
+// This test uses the system provided database. This makes the test portable,
+// but may cause failures when the database information changes. Historic data
+// may change if new facts are uncovered, future data may change when regions
+// change their time zone or daylight saving time. Most tests will not look in
+// the future to attempt to avoid issues. All tests list the data on which they
+// are based, this makes debugging easier upon failure; including to see whether
+// the provided data has not been changed.
+//
+// The first part of the test is manually crafted, the second part compares the
+// transitions for all time zones in the database.
+
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <format>
+
+#include "test_macros.h"
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+// The year range to validate. The dates used in practice are expected to be
+// inside the tested range.
+constexpr std::chrono::year first{1800};
+constexpr std::chrono::year last{2100};
+
+/***** ***** HELPERS ***** *****/
+
+[[nodiscard]] static std::chrono::sys_seconds to_sys_seconds(
+    std::chrono::year year,
+    std::chrono::month month,
+    std::chrono::day day,
+    std::chrono::hours h   = std::chrono::hours(0),
+    std::chrono::minutes m = std::chrono::minutes{0},
+    std::chrono::seconds s = std::chrono::seconds{0}) {
+  std::chrono::year_month_day result{year, month, day};
+
+  return std::chrono::time_point_cast<std::chrono::seconds>(static_cast<std::chrono::sys_days>(result)) + h + m + s;
+}
+
+[[nodiscard]] static std::chrono::local_seconds to_local_seconds(
+    std::chrono::year year,
+    std::chrono::month month,
+    std::chrono::day day,
+    std::chrono::hours h   = std::chrono::hours(0),
+    std::chrono::minutes m = std::chrono::minutes{0},
+    std::chrono::seconds s = std::chrono::seconds{0}) {
+  std::chrono::year_month_day result{year, month, day};
+
+  return std::chrono::time_point_cast<std::chrono::seconds>(static_cast<std::chrono::local_days>(result)) + h + m + s;
+}
+
+static void assert_equal(const std::chrono::sys_info& lhs, const std::chrono::sys_info& rhs) {
+  TEST_REQUIRE(lhs.begin == rhs.begin,
+               TEST_WRITE_CONCATENATED("\nBegin:\nExpected output ", lhs.begin, "\nActual output   ", rhs.begin, '\n'));
+  TEST_REQUIRE(lhs.end == rhs.end,
+               TEST_WRITE_CONCATENATED("\nEnd:\nExpected output ", lhs.end, "\nActual output   ", rhs.end, '\n'));
+  TEST_REQUIRE(
+      lhs.offset == rhs.offset,
+      TEST_WRITE_CONCATENATED("\nOffset:\nExpected output ", lhs.offset, "\nActual output   ", rhs.offset, '\n'));
+  TEST_REQUIRE(lhs.save == rhs.save,
+               TEST_WRITE_CONCATENATED("\nSave:\nExpected output ", lhs.save, "\nActual output   ", rhs.save, '\n'));
+  TEST_REQUIRE(
+      lhs.abbrev == rhs.abbrev,
+      TEST_WRITE_CONCATENATED("\nAbbrev:\nExpected output ", lhs.abbrev, "\nActual output   ", rhs.abbrev, '\n'));
+}
+
+static void assert_equal(const std::chrono::local_info& lhs, const std::chrono::local_info& rhs) {
+  TEST_REQUIRE(
+      lhs.result == rhs.result,
+      TEST_WRITE_CONCATENATED("\nResult:\nExpected output ", lhs.result, "\nActual output   ", rhs.result, '\n'));
+
+  assert_equal(lhs.first, rhs.first);
+  assert_equal(lhs.second, rhs.second);
+}
+
+/***** ***** TESTS ***** *****/
+
+static void test_gmt() {
+  // Simple zone always valid, no rule entries, lookup using a link.
+  // L Etc/GMT GMT
+  // Z Etc/GMT 0 - GMT
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("GMT");
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), 0s, 0min, "GMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+}
+
+static void test_local_time_out_of_range() {
+  // Fixed positive offset
+  // Etc/GMT-1 1 - +01
+
+  using namespace std::literals::chrono_literals;
+  { // lower bound
+    const std::chrono::time_zone* tz = std::chrono::locate_zone("Etc/GMT-1");
+
+    assert_equal(
+        std::chrono::local_info(
+            -1,
+            std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), 1h, 0min, "+01"),
+            std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+        tz->get_info(std::chrono::local_seconds::min()));
+
+    assert_equal(
+        std::chrono::local_info(
+            -1,
+            std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), 1h, 0min, "+01"),
+            std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+        tz->get_info(std::chrono::local_seconds::min() + 59min + 59s));
+
+    assert_equal(
+        std::chrono::local_info(
+            std::chrono::local_info::unique,
+            std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), 1h, 0min, "+01"),
+            std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+        tz->get_info(std::chrono::local_seconds::min() + 1h));
+  }
+
+  { // upper bound
+    const std::chrono::time_zone* tz = std::chrono::locate_zone("Etc/GMT+1");
+
+    assert_equal(
+        std::chrono::local_info(
+            -2,
+            std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), -1h, 0min, "-01"),
+            std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+        tz->get_info(std::chrono::local_seconds::max() - 1s));
+
+    assert_equal(
+        std::chrono::local_info(
+            std::chrono::local_info::unique,
+            std::chrono::sys_info(std::chrono::sys_seconds::min(), std::chrono::sys_seconds::max(), -1h, 0min, "-01"),
+            std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+        tz->get_info(std::chrono::local_seconds::max() - 1h - 1s));
+  }
+}
+
+static void test_indian_kerguelen() {
+  // One change, no rules, no dst changes.
+
+  // Z Indian/Kerguelen 0 - -00 1950
+  // 5 - +05
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Indian/Kerguelen");
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1950y, std::chrono::January, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1950y, std::chrono::January, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(
+              to_sys_seconds(1950y, std::chrono::January, 1d), std::chrono::sys_seconds::max(), 5h, 0min, "+05")),
+      tz->get_info(to_local_seconds(1950y, std::chrono::January, 1d)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1950y, std::chrono::January, 1d), std::chrono::sys_seconds::max(), 5h, 0min, "+05"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1950y, std::chrono::January, 1d, 5h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1950y, std::chrono::January, 1d), std::chrono::sys_seconds::max(), 5h, 0min, "+05"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::max() - 1s));
+}
+
+static void test_antarctica_rothera() {
+  // One change, no rules, no dst changes
+
+  // Z Antarctica/Rothera 0 - -00 1976 D
+  // -3 - -03
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Antarctica/Rothera");
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1976y, std::chrono::December, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1976y, std::chrono::December, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1976y, std::chrono::November, 30d, 20h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1976y, std::chrono::December, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(
+              to_sys_seconds(1976y, std::chrono::December, 1d), std::chrono::sys_seconds::max(), -3h, 0min, "-03")),
+      tz->get_info(to_local_seconds(1976y, std::chrono::November, 30d, 21h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(), to_sys_seconds(1976y, std::chrono::December, 1d), 0s, 0min, "-00"),
+          std::chrono::sys_info(
+              to_sys_seconds(1976y, std::chrono::December, 1d), std::chrono::sys_seconds::max(), -3h, 0min, "-03")),
+      tz->get_info(to_local_seconds(1976y, std::chrono::November, 30d, 23h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1976y, std::chrono::December, 1d), std::chrono::sys_seconds::max(), -3h, 0min, "-03"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1976y, std::chrono::December, 1d)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1976y, std::chrono::December, 1d), std::chrono::sys_seconds::max(), -3h, 0min, "-03"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::max() - 3h - 1s));
+
+  assert_equal(
+      std::chrono::local_info(
+          -2,
+          std::chrono::sys_info(
+              to_sys_seconds(1976y, std::chrono::December, 1d), std::chrono::sys_seconds::max(), -3h, 0min, "-03"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::max() - 1s));
+}
+
+static void test_asia_hong_kong() {
+  // A more typical entry, first some hard-coded entires and then at the
+  // end a rules based entry. This rule is valid for its entire period
+  //
+  // Z Asia/Hong_Kong 7:36:42 - LMT 1904 O 30 0:36:42
+  // 8 - HKT 1941 Jun 15 3
+  // 8 1 HKST 1941 O 1 4
+  // 8 0:30 HKWT 1941 D 25
+  // 9 - JST 1945 N 18 2
+  // 8 HK HK%sT
+  //
+  // R HK 1946 o - Ap 21 0 1 S
+  // R HK 1946 o - D 1 3:30s 0 -
+  // R HK 1947 o - Ap 13 3:30s 1 S
+  // R HK 1947 o - N 30 3:30s 0 -
+  // R HK 1948 o - May 2 3:30s 1 S
+  // R HK 1948 1952 - O Su>=28 3:30s 0 -
+  // R HK 1949 1953 - Ap Su>=1 3:30 1 S
+  // R HK 1953 1964 - O Su>=31 3:30 0 -
+  // R HK 1954 1964 - Mar Su>=18 3:30 1 S
+  // R HK 1965 1976 - Ap Su>=16 3:30 1 S
+  // R HK 1965 1976 - O Su>=16 3:30 0 -
+  // R HK 1973 o - D 30 3:30 1 S
+  // R HK 1979 o - May 13 3:30 1 S
+  // R HK 1979 o - O 21 3:30 0 -
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Asia/Hong_Kong");
+
+  assert_equal(
+      std::chrono::local_info(
+          -1,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          -1,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min() + 7h + 36min + 41s));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min() + 7h + 36min + 42s));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1904y, std::chrono::October, 30d, 0h, 36min, 41s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT")),
+      tz->get_info(to_local_seconds(1904y, std::chrono::October, 30d, 0h, 36min, 42s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              7h + 36min + 42s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT")),
+      tz->get_info(to_local_seconds(1904y, std::chrono::October, 30d, 0h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1904y, std::chrono::October, 30d, 1h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::June, 15d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::June, 15d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1904y, std::chrono::October, 29d, 17h),
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              8h,
+              0min,
+              "HKT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::June, 15d, 3h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::June, 15d, 4h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::October, 1d, 3h, 29min, 29s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              to_sys_seconds(1941y, std::chrono::December, 24d, 15h, 30min),
+              8h + 30min,
+              30min,
+              "HKWT")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::October, 1d, 3h, 30min)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::June, 14d, 19h),
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              9h,
+              60min,
+              "HKST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              to_sys_seconds(1941y, std::chrono::December, 24d, 15h, 30min),
+              8h + 30min,
+              30min,
+              "HKWT")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::October, 1d, 3h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1941y, std::chrono::September, 30d, 19h),
+              to_sys_seconds(1941y, std::chrono::December, 24d, 15h, 30min),
+              8h + 30min,
+              30min,
+              "HKWT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1941y, std::chrono::October, 1d, 4h)));
+}
+
+static void test_europe_berlin() {
+  // A more typical entry, first some hard-coded entires and then at the
+  // end a rules based entry. This rule is valid for its entire period
+  //
+
+  // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+  // 1 c CE%sT 1945 May 24 2
+  // 1 So CE%sT 1946
+  // 1 DE CE%sT 1980
+  // 1 E CE%sT
+  //
+  // R c 1916 o - Ap 30 23 1 S
+  // R c 1916 o - O 1 1 0 -
+  // R c 1917 1918 - Ap M>=15 2s 1 S
+  // R c 1917 1918 - S M>=15 2s 0 -
+  // R c 1940 o - Ap 1 2s 1 S
+  // R c 1942 o - N 2 2s 0 -
+  // R c 1943 o - Mar 29 2s 1 S
+  // R c 1943 o - O 4 2s 0 -
+  // R c 1944 1945 - Ap M>=1 2s 1 S
+  // R c 1944 o - O 2 2s 0 -
+  // R c 1945 o - S 16 2s 0 -
+  // R c 1977 1980 - Ap Su>=1 2s 1 S
+  // R c 1977 o - S lastSu 2s 0 -
+  // R c 1978 o - O 1 2s 0 -
+  // R c 1979 1995 - S lastSu 2s 0 -
+  // R c 1981 ma - Mar lastSu 2s 1 S
+  // R c 1996 ma - O lastSu 2s 0 -
+  //
+  // R So 1945 o - May 24 2 2 M
+  // R So 1945 o - S 24 3 1 S
+  // R So 1945 o - N 18 2s 0 -
+  //
+  // R DE 1946 o - Ap 14 2s 1 S
+  // R DE 1946 o - O 7 2s 0 -
+  // R DE 1947 1949 - O Su>=1 2s 0 -
+  // R DE 1947 o - Ap 6 3s 1 S
+  // R DE 1947 o - May 11 2s 2 M
+  // R DE 1947 o - Jun 29 3 1 S
+  // R DE 1948 o - Ap 18 2s 1 S
+  // R DE 1949 o - Ap 10 2s 1 S
+  //
+  // R E 1977 1980 - Ap Su>=1 1u 1 S
+  // R E 1977 o - S lastSu 1u 0 -
+  // R E 1978 o - O 1 1u 0 -
+  // R E 1979 1995 - S lastSu 1u 0 -
+  // R E 1981 ma - Mar lastSu 1u 1 S
+  // R E 1996 ma - O lastSu 1u 0 -
+  //
+  // Note the European Union decided to stop the seasonal change in
+  // 2021. In 2023 seasonal changes are still in effect.
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Berlin");
+
+  assert_equal(
+      std::chrono::local_info(
+          -1,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1893y, std::chrono::March, 31d, 23h, 6min, 32s),
+              53min + 28s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          -1,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1893y, std::chrono::March, 31d, 23h, 6min, 32s),
+              53min + 28s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min() + 53min + 27s));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1893y, std::chrono::March, 31d, 23h, 6min, 32s),
+              53min + 28s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min() + 53min + 28s));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1893y, std::chrono::March, 31d, 23h, 6min, 32s),
+              53min + 28s,
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1893y, std::chrono::March, 31d, 23h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1946y, std::chrono::October, 7d, 1h),
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              1h,
+              0min,
+              "CET"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::April, 6d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1946y, std::chrono::October, 7d, 1h),
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              1h,
+              0min,
+              "CET"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::April, 6d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1946y, std::chrono::October, 7d, 1h),
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              1h,
+              0min,
+              "CET"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::April, 6d, 3h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::April, 6d, 4h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::May, 11d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::May, 11d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::April, 6d, 2h),
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::May, 11d, 3h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::May, 11d, 4h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::June, 29d, 1h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::June, 29d, 2h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::May, 11d, 1h),
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              3h,
+              120min,
+              "CEMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::June, 29d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::June, 29d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::October, 5d, 1h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              to_sys_seconds(1948y, std::chrono::April, 18d, 1h),
+              1h,
+              0min,
+              "CET")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::October, 5d, 2h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::June, 29d),
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              2h,
+              60min,
+              "CEST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              to_sys_seconds(1948y, std::chrono::April, 18d, 1h),
+              1h,
+              0min,
+              "CET")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::October, 5d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1947y, std::chrono::October, 5d, 1h),
+              to_sys_seconds(1948y, std::chrono::April, 18d, 1h),
+              1h,
+              0min,
+              "CET"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1947y, std::chrono::October, 5d, 3h)));
+}
+
+static void test_europe_dublin() {
+  // Z Europe/Dublin -0:25:21 - LMT 1880 Au 2
+  // -0:25:21 - DMT 1916 May 21 2s
+  // -0:25:21 1 IST 1916 O 1 2s
+  // 0 G %s 1921 D 6
+  // ...
+  //
+  // R G 1916 o - May 21 2s 1 BST
+  // R G 1916 o - O 1 2s 0 GMT
+  // R G 1917 o - Ap 8 2s 1 BST
+  // ...
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Dublin");
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1880y, std::chrono::August, 1d, 23h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "DMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1880y, std::chrono::August, 2d)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "DMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::May, 21d, 1h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "DMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              34min + 39s,
+              60min,
+              "IST")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::May, 21d, 2h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1880y, std::chrono::August, 2d, 0h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              -(25min + 21s),
+              0min,
+              "DMT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              34min + 39s,
+              60min,
+              "IST")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::May, 21d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              34min + 39s,
+              60min,
+              "IST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::May, 21d, 6h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              34min + 39s,
+              60min,
+              "IST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::October, 1d, 2h, 25min, 20s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::May, 21d, 2h, 25min, 21s),
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              34min + 39s,
+              60min,
+              "IST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 2h),
+              0s,
+              0min,
+              "GMT")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::October, 1d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 2h),
+              0s,
+              0min,
+              "GMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1916y, std::chrono::October, 1d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1916y, std::chrono::October, 1d, 02h, 25min, 21s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 2h),
+              0s,
+              0min,
+              "GMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::April, 8d, 1h, 59min, 59s)));
+}
+
+static void test_america_st_johns() {
+  // A more typical entry,
+  // Uses letters both when DST is ative and not and has multiple
+  // letters. Uses negetive offsets.
+  // Switches several times between their own and Canadian rules
+  // Switches the stdoff from -3:30:52 to -3:30 while observing the same rule
+
+  // Z America/St_Johns -3:30:52 - LMT 1884
+  // -3:30:52 j N%sT 1918
+  // -3:30:52 C N%sT 1919
+  // ...
+  //
+  // R j 1917 o - Ap 8 2 1 D
+  // R j 1917 o - S 17 2 0 S
+  // R j 1919 o - May 5 23 1 D
+  // R j 1919 o - Au 12 23 0 S
+  // R j 1920 1935 - May Su>=1 23 1 D
+  // ...
+  //
+  // R C 1918 o - Ap 14 2 1 D
+  // R C 1918 o - O 27 2 0 S
+  // R C 1942 o - F 9 2 1 W
+  // ...
+
+  using namespace std::literals::chrono_literals;
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("America/St_Johns");
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(std::chrono::local_seconds::min()));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              std::chrono::sys_seconds::min(),
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "LMT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1883y, std::chrono::December, 31d, 23h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1884y, std::chrono::January, 1d)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::April, 8d, 1h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::April, 8d, 2h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::nonexistent,
+          std::chrono::sys_info(
+              to_sys_seconds(1884y, std::chrono::January, 1d, 3h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST"),
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::April, 8d, 2h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::April, 8d, 3h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::September, 17d, 0h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              to_sys_seconds(1918y, std::chrono::April, 14d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::September, 17d, 1h)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::ambiguous,
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::April, 8d, 5h, 30min, 52s),
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              -(2h + 30min + 52s),
+              60min,
+              "NDT"),
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              to_sys_seconds(1918y, std::chrono::April, 14d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::September, 17d, 1h, 59min, 59s)));
+
+  assert_equal(
+      std::chrono::local_info(
+          std::chrono::local_info::unique,
+          std::chrono::sys_info(
+              to_sys_seconds(1917y, std::chrono::September, 17d, 4h, 30min, 52s),
+              to_sys_seconds(1918y, std::chrono::April, 14d, 5h, 30min, 52s),
+              -(3h + 30min + 52s),
+              0min,
+              "NST"),
+          std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+      tz->get_info(to_local_seconds(1917y, std::chrono::September, 17d, 2h)));
+}
+
+static void validate_transitions(const std::chrono::time_zone& zone) {
+  using namespace std::literals::chrono_literals;
+
+  constexpr auto begin = std::chrono::time_point_cast<std::chrono::seconds>(
+      static_cast<std::chrono::sys_days>(std::chrono::year_month_day{first, std::chrono::January, 1d}));
+  constexpr auto end = std::chrono::time_point_cast<std::chrono::seconds>(
+      static_cast<std::chrono::sys_days>(std::chrono::year_month_day{last, std::chrono::January, 1d}));
+
+  // Builds the set of sys_info objects for the selected time range.
+  std::vector<std::chrono::sys_info> input;
+  std::chrono::sys_seconds s = begin;
+  do {
+    input.emplace_back(zone.get_info(s));
+    s = input.back().end;
+  } while (s < end);
+
+  for (auto previous = input.begin(), next = previous + 1; next != input.end(); ++previous, ++next) {
+    // Now iterates to all adjacent objects.
+    // For every transition gets the locate time of the
+    // - end of the first          (a)
+    // - the start if the second   (b)
+    // Depending on the difference between 'a' and 'b' different tests are done.
+    std::chrono::local_seconds end_previous{previous->end.time_since_epoch() + previous->offset};
+    std::chrono::local_seconds begin_next{next->begin.time_since_epoch() + next->offset};
+
+    if (end_previous == begin_next) {
+      // unique transition
+      // a |------------|
+      // b              |----------|
+      //                T
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *previous,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(end_previous - 1s));
+
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *next,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(begin_next));
+
+    } else if (end_previous < begin_next) {
+      // non-existent transition
+      // a |------------|
+      // b                 |----------|
+      //                T  T
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *previous,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(end_previous - 1s));
+
+      assert_equal(std::chrono::local_info(std::chrono::local_info::nonexistent, *previous, *next),
+                   zone.get_info(end_previous));
+
+      assert_equal(std::chrono::local_info(std::chrono::local_info::nonexistent, *previous, *next),
+                   zone.get_info(begin_next - 1s));
+
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *next,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(begin_next));
+
+    } else {
+      // ambiguous transition
+      // a |------------|
+      // b           |----------|
+      //             T  T
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *previous,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(begin_next - 1s));
+
+      assert_equal(std::chrono::local_info(std::chrono::local_info::ambiguous, *previous, *next),
+                   zone.get_info(begin_next));
+
+      assert_equal(std::chrono::local_info(std::chrono::local_info::ambiguous, *previous, *next),
+                   zone.get_info(end_previous - 1s));
+
+      assert_equal(std::chrono::local_info(
+                       std::chrono::local_info::unique,
+                       *next,
+                       std::chrono::sys_info(std::chrono::sys_seconds(0s), std::chrono::sys_seconds(0s), 0s, 0min, "")),
+                   zone.get_info(end_previous));
+    }
+  }
+}
+
+int main(int, const char**) {
+  test_gmt();
+  test_local_time_out_of_range();
+  test_indian_kerguelen();
+  test_antarctica_rothera();
+
+  test_asia_hong_kong();
+  test_europe_berlin();
+  test_europe_dublin();
+  test_america_st_johns();
+
+  const std::chrono::tzdb& tzdb = std::chrono::get_tzdb();
+  for (const auto& zone : tzdb.zones) {
+    validate_transitions(zone);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_local.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_local.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..28d61455710cb10cbe6262fc51be7fbf2fc076a2
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_local.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class time_zone;
+
+// template <class _Duration>
+// local_time<common_type_t<Duration, seconds>>
+//   to_local(const sys_time<Duration>& tp) const;
+
+#include <chrono>
+#include <format>
+#include <cassert>
+#include <string_view>
+
+#include "test_macros.h"
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+int main(int, char**) {
+  // To make sure the test does not depend on changes in the database it uses a
+  // time zone with a fixed offset.
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Etc/GMT+1");
+
+  assert(tz->to_local(std::chrono::sys_time<std::chrono::nanoseconds>{-1ns}) ==
+         std::chrono::local_time<std::chrono::nanoseconds>{-1ns - 1h});
+
+  assert(tz->to_local(std::chrono::sys_time<std::chrono::microseconds>{0us}) ==
+         std::chrono::local_time<std::chrono::microseconds>{0us - 1h});
+
+  assert(tz->to_local(
+             std::chrono::sys_time<std::chrono::seconds>{std::chrono::sys_days{std::chrono::January / 1 / -21970}}) ==
+         std::chrono::local_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch() - 1h});
+
+  assert(
+      tz->to_local(std::chrono::sys_time<std::chrono::days>{std::chrono::sys_days{std::chrono::January / 1 / 21970}}) ==
+      std::chrono::local_time<std::chrono::seconds>{
+          (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch() - 1h});
+
+  assert(tz->to_local(std::chrono::sys_time<std::chrono::weeks>{}) ==
+         std::chrono::local_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() - 1h});
+
+  assert(tz->to_local(std::chrono::sys_time<std::chrono::months>{}) ==
+         std::chrono::local_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() - 1h});
+
+  assert(tz->to_local(std::chrono::sys_time<std::chrono::years>{}) ==
+         std::chrono::local_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() - 1h});
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..874c3d52e460b1bab6bd6e88ce5971b7cbe935a6
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys.pass.cpp
@@ -0,0 +1,237 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class time_zone;
+
+// template <class _Duration>
+// sys_time<common_type_t<Duration, seconds>>
+//   to_sys(const local_time<Duration>& tp) const;
+
+#include <chrono>
+#include <format>
+#include <cassert>
+#include <string_view>
+
+#include "test_macros.h"
+#include "assert_macros.h"
+#include "concat_macros.h"
+
+// Tests unique conversions. To make sure the test is does not depend on changes
+// in the database it uses a time zone with a fixed offset.
+static void test_unique() {
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Etc/GMT+1");
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::nanoseconds>{-1ns}) ==
+         std::chrono::sys_time<std::chrono::nanoseconds>{-1ns + 1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::microseconds>{0us}) ==
+         std::chrono::sys_time<std::chrono::microseconds>{1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch()}) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch() + 1h});
+
+  // sys_time<common_type_t<Duration, seconds>> is seconds for the larger types
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::days>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch()}) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch() + 1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::weeks>{}) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() + 1h});
+
+  // Note months and years can not be streamed; thus the function cannot be
+  // instantiated for these types. (Even when there is no exception thrown.)
+}
+
+// Tests non-existant conversions.
+static void test_nonexistent() {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Berlin");
+
+  // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+  // ...
+  // 1 DE CE%sT 1980
+  // 1 E CE%sT
+  //
+  // ...
+  // R E 1981 ma - Mar lastSu 1u 1 S
+  // R E 1996 ma - O lastSu 1u 0 -
+
+  // Pick an historic date where it's well known what the time zone rules were.
+  // This makes it unlikely updates to the database change these rules.
+  std::chrono::local_time<std::chrono::seconds> time{
+      (std::chrono::sys_days{std::chrono::March / 30 / 1986} + 2h + 30min).time_since_epoch()};
+
+  // Validates whether the database did not change.
+  std::chrono::local_info info = tz->get_info(time);
+  assert(info.result == std::chrono::local_info::nonexistent);
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::nonexistent_local_time,
+      [&]([[maybe_unused]] const std::chrono::nonexistent_local_time& e) {
+        std::string_view what =
+            R"(1986-03-30 02:30:00.000000000 is in a gap between
+1986-03-30 02:00:00 CET and
+1986-03-30 03:00:00 CEST which are both equivalent to
+1986-03-30 01:00:00 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0ns));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::nonexistent_local_time,
+      [&]([[maybe_unused]] const std::chrono::nonexistent_local_time& e) {
+        std::string_view what =
+            R"(1986-03-30 02:30:00.000000 is in a gap between
+1986-03-30 02:00:00 CET and
+1986-03-30 03:00:00 CEST which are both equivalent to
+1986-03-30 01:00:00 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0us));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::nonexistent_local_time,
+      [&]([[maybe_unused]] const std::chrono::nonexistent_local_time& e) {
+        std::string_view what =
+            R"(1986-03-30 02:30:00.000 is in a gap between
+1986-03-30 02:00:00 CET and
+1986-03-30 03:00:00 CEST which are both equivalent to
+1986-03-30 01:00:00 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0ms));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::nonexistent_local_time,
+      [&]([[maybe_unused]] const std::chrono::nonexistent_local_time& e) {
+        std::string_view what =
+            R"(1986-03-30 02:30:00 is in a gap between
+1986-03-30 02:00:00 CET and
+1986-03-30 03:00:00 CEST which are both equivalent to
+1986-03-30 01:00:00 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0s));
+
+#endif // TEST_HAS_NO_EXCEPTIONS
+}
+
+// Tests ambiguous conversions.
+static void test_ambiguous() {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Berlin");
+
+  // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+  // ...
+  // 1 DE CE%sT 1980
+  // 1 E CE%sT
+  //
+  // ...
+  // R E 1981 ma - Mar lastSu 1u 1 S
+  // R E 1996 ma - O lastSu 1u 0 -
+
+  // Pick an historic date where it's well known what the time zone rules were.
+  // This makes it unlikely updates to the database change these rules.
+  std::chrono::local_time<std::chrono::seconds> time{
+      (std::chrono::sys_days{std::chrono::September / 28 / 1986} + 2h + 30min).time_since_epoch()};
+
+  // Validates whether the database did not change.
+  std::chrono::local_info info = tz->get_info(time);
+  assert(info.result == std::chrono::local_info::ambiguous);
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::ambiguous_local_time,
+      [&]([[maybe_unused]] const std::chrono::ambiguous_local_time& e) {
+        std::string_view what =
+            R"(1986-09-28 02:30:00.000000000 is ambiguous.  It could be
+1986-09-28 02:30:00.000000000 CEST == 1986-09-28 00:30:00.000000000 UTC or
+1986-09-28 02:30:00.000000000 CET == 1986-09-28 01:30:00.000000000 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0ns));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::ambiguous_local_time,
+      [&]([[maybe_unused]] const std::chrono::ambiguous_local_time& e) {
+        std::string_view what =
+            R"(1986-09-28 02:30:00.000000 is ambiguous.  It could be
+1986-09-28 02:30:00.000000 CEST == 1986-09-28 00:30:00.000000 UTC or
+1986-09-28 02:30:00.000000 CET == 1986-09-28 01:30:00.000000 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0us));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::ambiguous_local_time,
+      [&]([[maybe_unused]] const std::chrono::ambiguous_local_time& e) {
+        std::string_view what =
+            R"(1986-09-28 02:30:00.000 is ambiguous.  It could be
+1986-09-28 02:30:00.000 CEST == 1986-09-28 00:30:00.000 UTC or
+1986-09-28 02:30:00.000 CET == 1986-09-28 01:30:00.000 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0ms));
+
+  TEST_VALIDATE_EXCEPTION(
+      std::chrono::ambiguous_local_time,
+      [&]([[maybe_unused]] const std::chrono::ambiguous_local_time& e) {
+        std::string_view what =
+            R"(1986-09-28 02:30:00 is ambiguous.  It could be
+1986-09-28 02:30:00 CEST == 1986-09-28 00:30:00 UTC or
+1986-09-28 02:30:00 CET == 1986-09-28 01:30:00 UTC)";
+        TEST_LIBCPP_REQUIRE(
+            e.what() == what,
+            TEST_WRITE_CONCATENATED("Expected exception\n", what, "\n\nActual exception\n", e.what(), '\n'));
+      },
+      tz->to_sys(time + 0s));
+
+#endif // TEST_HAS_NO_EXCEPTIONS
+}
+
+// This test does the basic validations of this function. The library function
+// uses `local_info get_info(const local_time<Duration>& tp)` as implementation
+// detail. The get_info function does extensive testing of the data.
+int main(int, char**) {
+  test_unique();
+  test_nonexistent();
+  test_ambiguous();
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bad4ef352e9b91772f9d422dae0094c34fff8019
--- /dev/null
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp
@@ -0,0 +1,147 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+
+// XFAIL: libcpp-has-no-experimental-tzdb
+// XFAIL: availability-tzdb-missing
+
+// <chrono>
+
+// class time_zone;
+
+// template <class _Duration>
+//   sys_time<common_type_t<Duration, seconds>>
+//     to_sys(const local_time<Duration>& tp, choose z) const;
+
+#include <chrono>
+#include <format>
+#include <cassert>
+#include <string_view>
+
+#include "test_macros.h"
+
+// Tests unique conversions. To make sure the test is does not depend on changes
+// in the database it uses a time zone with a fixed offset.
+static void test_unique() {
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Etc/GMT+1");
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::nanoseconds>{-1ns}, std::chrono::choose::earliest) ==
+         std::chrono::sys_time<std::chrono::nanoseconds>{-1ns + 1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::microseconds>{0us}, std::chrono::choose::latest) ==
+         std::chrono::sys_time<std::chrono::microseconds>{1h});
+
+  assert(tz->to_sys(
+             std::chrono::local_time<std::chrono::seconds>{
+                 (std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch()},
+             std::chrono::choose::earliest) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / -21970}).time_since_epoch() + 1h});
+
+  // sys_time<common_type_t<Duration, seconds>> is seconds for the larger types
+  assert(tz->to_sys(
+             std::chrono::local_time<std::chrono::days>{
+                 (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch()},
+             std::chrono::choose::latest) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 21970}).time_since_epoch() + 1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::weeks>{}, std::chrono::choose::earliest) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() + 1h});
+
+  // Note months and years cannot be streamed; however these functions don't
+  // throw an exception and thus can be used.
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::months>{}, std::chrono::choose::latest) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() + 1h});
+
+  assert(tz->to_sys(std::chrono::local_time<std::chrono::years>{}, std::chrono::choose::earliest) ==
+         std::chrono::sys_time<std::chrono::seconds>{
+             (std::chrono::sys_days{std::chrono::January / 1 / 1970}).time_since_epoch() + 1h});
+}
+
+// Tests non-existant conversions.
+static void test_nonexistent() {
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Berlin");
+
+  // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+  // ...
+  // 1 DE CE%sT 1980
+  // 1 E CE%sT
+  //
+  // ...
+  // R E 1981 ma - Mar lastSu 1u 1 S
+  // R E 1996 ma - O lastSu 1u 0 -
+
+  // Pick an historic date where it's well known what the time zone rules were.
+  // This makes it unlikely updates to the database change these rules.
+  std::chrono::local_time<std::chrono::seconds> time{
+      (std::chrono::sys_days{std::chrono::March / 30 / 1986} + 2h + 30min).time_since_epoch()};
+
+  std::chrono::sys_seconds expected{time.time_since_epoch() - 1h};
+
+  // Validates whether the database did not change.
+  std::chrono::local_info info = tz->get_info(time);
+  assert(info.result == std::chrono::local_info::nonexistent);
+
+  assert(tz->to_sys(time + 0ns, std::chrono::choose::earliest) == expected);
+  assert(tz->to_sys(time + 0us, std::chrono::choose::latest) == expected);
+  assert(tz->to_sys(time + 0ms, std::chrono::choose::earliest) == expected);
+  assert(tz->to_sys(time + 0s, std::chrono::choose::latest) == expected);
+}
+
+// Tests ambiguous conversions.
+static void test_ambiguous() {
+  using namespace std::literals::chrono_literals;
+
+  const std::chrono::time_zone* tz = std::chrono::locate_zone("Europe/Berlin");
+
+  // Z Europe/Berlin 0:53:28 - LMT 1893 Ap
+  // ...
+  // 1 DE CE%sT 1980
+  // 1 E CE%sT
+  //
+  // ...
+  // R E 1981 ma - Mar lastSu 1u 1 S
+  // R E 1996 ma - O lastSu 1u 0 -
+
+  // Pick an historic date where it's well known what the time zone rules were.
+  // This makes it unlikely updates to the database change these rules.
+  std::chrono::local_time<std::chrono::seconds> time{
+      (std::chrono::sys_days{std::chrono::September / 28 / 1986} + 2h + 30min).time_since_epoch()};
+
+  std::chrono::sys_seconds earlier{time.time_since_epoch() - 2h};
+  std::chrono::sys_seconds later{time.time_since_epoch() - 1h};
+
+  // Validates whether the database did not change.
+  std::chrono::local_info info = tz->get_info(time);
+  assert(info.result == std::chrono::local_info::ambiguous);
+
+  assert(tz->to_sys(time + 0ns, std::chrono::choose::earliest) == earlier);
+  assert(tz->to_sys(time + 0us, std::chrono::choose::latest) == later);
+  assert(tz->to_sys(time + 0ms, std::chrono::choose::earliest) == earlier);
+  assert(tz->to_sys(time + 0s, std::chrono::choose::latest) == later);
+}
+
+// This test does the basic validations of this function. The library function
+// uses `local_info get_info(const local_time<Duration>& tp)` as implementation
+// detail. The get_info function does extensive testing of the data.
+int main(int, char**) {
+  test_unique();
+  test_nonexistent();
+  test_ambiguous();
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
index ad9d2a9d9ccc00b13675856438f4102b56d02c54..37be6ceea3e094b7f08c790289a6ba41617f6568 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -17,7 +15,7 @@
 // template <class T>
 // bool
 // atomic_compare_exchange_strong(shared_ptr<T>* p, shared_ptr<T>* v,
-//                                shared_ptr<T> w);                     // Deprecated in C++20, removed in C++26
+//                                shared_ptr<T> w);
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
index 26496df96d472d8bc005848922a55a869a4e1296..3965863b86ccf540f7fa71f55e313ddb0d524cdc 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -18,7 +16,7 @@
 // bool
 // atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
 //                                         shared_ptr<T> w, memory_order success,
-//                                         memory_order failure);                 // Deprecated in C++20, removed in C++26
+//                                         memory_order failure);
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
index 5cdc0ea3312ff5e05be7a07b5a3266fa1d94abc3..6dd04f924a105ede8821de31dc1d2610215bf31c 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -17,7 +15,7 @@
 // template <class T>
 // bool
 // atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v,
-//                              shared_ptr<T> w);                    // Deprecated in C++20, removed in C++26
+//                              shared_ptr<T> w);
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
index e4cc4cf77a64e2a6e51ba47fc63a75600033bcb9..4837fa9793a5e874387e9cc70be9722ac01489b6 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -18,7 +16,7 @@
 // bool
 // atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
 //                                       shared_ptr<T> w, memory_order success,
-//                                       memory_order failure);                 // Deprecated in C++20, removed in C++26
+//                                       memory_order failure);
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
index db02c9d9f75b4247500378032c416e73e70c0e1e..f488e0ed1d14e13446d07d5b7bf5f9136ed92016 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
+// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
index 704232e71fa4aedb382ebf96373f3cb3ff76afa1..1945f7bba6dc060873f0096866ac37715c9423e5 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)  // Deprecated in C++20, removed in C++26
+// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
index 8f3b5bbb51558beabc57404ba86a142d539a5744..37f7d12eda2c626851ca59b64193c7b212b81a92 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template<class T>
 // bool
-// atomic_is_lock_free(const shared_ptr<T>* p);    // Deprecated in C++20, removed in C++26
+// atomic_is_lock_free(const shared_ptr<T>* p);
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
index 9d81e81b4392b86bdc928d302acc8c4f16b239cf..1b9a15ac92ac1fd795973d0bec89f83b74a7e42e 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_load(const shared_ptr<T>* p)      // Deprecated in C++20, removed in C++26
+// atomic_load(const shared_ptr<T>* p)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
index f4875dce38a81050edf80a4d33610a04f88759f3..5c2970133328f37683b56cdffcf3bfc97b59fc25 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)     // Deprecated in C++20, removed in C++26
+// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
index 836dcb16ff32e3a60dd340b7262cf0746cd89060..5b7bd5fad69c473336407935f1e08ebcb694e425 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // void
-// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
+// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
index 85b31c017d736226bf5431a4bf16df6486632b98..5712190421308d0eeba6479e2ce97946d5665de0 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
@@ -5,9 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+//
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +14,7 @@
 
 // template <class T>
 // void
-// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)  // Deprecated in C++20, removed in C++26
+// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxxabi/src/aix_state_tab_eh.inc b/libcxxabi/src/aix_state_tab_eh.inc
index 285c9ac71d60d5515f184a2c533f2a3fcf78f0e0..0ed329890a0a44531542c986bfc703abeea10f12 100644
--- a/libcxxabi/src/aix_state_tab_eh.inc
+++ b/libcxxabi/src/aix_state_tab_eh.inc
@@ -681,7 +681,7 @@ static uintptr_t* skip_non_cxx_eh_aware_frames(uint32_t* Pc, uintptr_t* Sp) {
 // xlclang++ compiled code. If __xlc_exception_handle() is called by
 // non-C++ EH aware functions, their frames are skipped until a C++ EH aware
 // frame is found.
-// Note: make sure __xlc_excpetion_handle() is a non-leaf function. Currently
+// Note: make sure __xlc_exception_handle() is a non-leaf function. Currently
 // it calls skip_non_cxx_eh_aware_frames(), which in turn calls abort().
 _LIBCXXABI_FUNC_VIS uintptr_t __xlc_exception_handle() {
   // Get the SP of this function, i.e., __xlc_exception_handle().
diff --git a/lld/COFF/Symbols.h b/lld/COFF/Symbols.h
index ca69fb2d052706d8dd7517450996a45a435ee096..5ef46f5af6a6c8d8396c36478334ed7b8ec049e4 100644
--- a/lld/COFF/Symbols.h
+++ b/lld/COFF/Symbols.h
@@ -106,7 +106,7 @@ protected:
            "If the name is empty, the Symbol must be a DefinedCOFF.");
   }
 
-  const unsigned symbolKind : 8;
+  unsigned symbolKind : 8;
   unsigned isExternal : 1;
 
 public:
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index cf5c2380690f147926ae0022bda0761dd1a657e4..47e6ea1ff775649ee344be373388ceade20d9d84 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -429,19 +429,6 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_PREL64:
     write64(loc, val);
     break;
-  case R_AARCH64_AUTH_ABS64:
-    // If val is wider than 32 bits, the relocation must have been moved from
-    // .relr.auth.dyn to .rela.dyn, and the addend write is not needed.
-    //
-    // If val fits in 32 bits, we have two potential scenarios:
-    // * True RELR: Write the 32-bit `val`.
-    // * RELA: Even if the value now fits in 32 bits, it might have been
-    //   converted from RELR during an iteration in
-    //   finalizeAddressDependentContent(). Writing the value is harmless
-    //   because dynamic linking ignores it.
-    if (isInt<32>(val))
-      write32(loc, val);
-    break;
   case R_AARCH64_ADD_ABS_LO12_NC:
     or32AArch64Imm(loc, val);
     break;
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 883c4a2f84294ceb51dd2513fd16b81444bafe70..0173be396163ebf5e2f9697bdc94d0ba6bbf0ccf 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -286,6 +286,7 @@ struct Config {
   bool relax;
   bool relaxGP;
   bool relocatable;
+  bool resolveGroups;
   bool relrGlibc = false;
   bool relrPackDynRelocs = false;
   llvm::DenseSet<llvm::StringRef> saveTempsArgs;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index ddc574a11314ba2f14f4e36c4fe6987e5e448521..ed214ab29d4c6f2aa33da0733648a83462b01c83 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1384,6 +1384,8 @@ static void readConfigs(opt::InputArgList &args) {
   config->relaxGP = args.hasFlag(OPT_relax_gp, OPT_no_relax_gp, false);
   config->rpath = getRpath(args);
   config->relocatable = args.hasArg(OPT_relocatable);
+  config->resolveGroups =
+      !args.hasArg(OPT_relocatable) || args.hasArg(OPT_force_group_allocation);
 
   if (args.hasArg(OPT_save_temps)) {
     // --save-temps implies saving all temps.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index d760dddcf5ec5c3619debb57dc5bad4303fed2e7..e6a0a5be821e07cca039fa0be9395ee4010889ab 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -676,7 +676,7 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
         symtab.comdatGroups.try_emplace(CachedHashStringRef(signature), this)
             .second;
     if (keepGroup) {
-      if (config->relocatable)
+      if (!config->resolveGroups)
         this->sections[i] = createInputSection(
             i, sec, check(obj.getSectionName(sec, shstrtab)));
       continue;
@@ -844,6 +844,16 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats,
       this->sections[i] =
           createInputSection(i, sec, check(obj.getSectionName(sec, shstrtab)));
       break;
+    case SHT_LLVM_LTO:
+      // Discard .llvm.lto in a relocatable link that does not use the bitcode.
+      // The concatenated output does not properly reflect the linking
+      // semantics. In addition, since we do not use the bitcode wrapper format,
+      // the concatenated raw bitcode would be invalid.
+      if (config->relocatable && !config->fatLTOObjects) {
+        sections[i] = &InputSection::discarded;
+        break;
+      }
+      [[fallthrough]];
     default:
       this->sections[i] =
           createInputSection(i, sec, check(obj.getSectionName(sec, shstrtab)));
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index e6c5996c0b392ac3af18495a0b47824d5dcf40c9..f4287bc94ee5ea70af9c1dd7d5b7026fde5b76f5 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -76,12 +76,12 @@ InputSectionBase::InputSectionBase(InputFile *file, uint64_t flags,
     invokeELFT(parseCompressedHeader,);
 }
 
-// Drop SHF_GROUP bit unless we are producing a re-linkable object file.
-// SHF_GROUP is a marker that a section belongs to some comdat group.
-// That flag doesn't make sense in an executable.
+// SHF_INFO_LINK and SHF_GROUP are normally resolved and not copied to the
+// output section. However, for relocatable linking without
+// --force-group-allocation, the SHF_GROUP flag and section groups are retained.
 static uint64_t getFlags(uint64_t flags) {
   flags &= ~(uint64_t)SHF_INFO_LINK;
-  if (!config->relocatable)
+  if (config->resolveGroups)
     flags &= ~(uint64_t)SHF_GROUP;
   return flags;
 }
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index ff61a566f52f73da27cc4e34403770add8d2433c..d10174681f5eb21a490681b211b9f0976c40b5d6 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -254,6 +254,9 @@ def fix_cortex_a53_843419: F<"fix-cortex-a53-843419">,
 def fix_cortex_a8: F<"fix-cortex-a8">,
   HelpText<"Apply fixes for ARM Cortex-A8 erratum 657417">;
 
+def force_group_allocation: FF<"force-group-allocation">,
+  HelpText<"Only meaningful for -r. Section groups are discarded. If two section group members are placed to the same output section, combine their relocations as well">;
+
 defm format: Eq<"format", "Change the input format of the inputs following this option">,
   MetaVarName<"[default,elf,binary]">;
 
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 2c02c2e572bfd03c54d8c2f6a1dfa7d5e8c226c5..04db413a6609fd83113d3213c71a207a259725eb 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -898,9 +898,9 @@ static void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec,
     isec.addReloc({expr, type, offsetInSec, addend, &sym});
     if (shard)
       part.relrDyn->relocsVec[parallel::getThreadIndex()].push_back(
-          {&isec, isec.relocs().size() - 1});
+          {&isec, offsetInSec});
     else
-      part.relrDyn->relocs.push_back({&isec, isec.relocs().size() - 1});
+      part.relrDyn->relocs.push_back({&isec, offsetInSec});
     return;
   }
   part.relaDyn->addRelativeReloc<shard>(target->relativeRel, isec, offsetInSec,
@@ -1154,12 +1154,6 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
         // relative relocation. Use a symbolic relocation instead.
         if (sym.isPreemptible) {
           part.relaDyn->addSymbolReloc(type, *sec, offset, sym, addend, type);
-        } else if (part.relrAuthDyn && sec->addralign >= 2 && offset % 2 == 0) {
-          // When symbol values are determined in
-          // finalizeAddressDependentContent, some .relr.auth.dyn relocations
-          // may be moved to .rela.dyn.
-          sec->addReloc({expr, type, offset, addend, &sym});
-          part.relrAuthDyn->relocs.push_back({sec, sec->relocs().size() - 1});
         } else {
           part.relaDyn->addReloc({R_AARCH64_AUTH_RELATIVE, sec, offset,
                                   DynamicReloc::AddendOnlyWithTargetVA, sym,
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index ad280289cebf9b76258fe95071ec74857e7e0424..cc423d152e9127e8f0ea060cc4386f311577acdf 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -1420,12 +1420,6 @@ DynamicSection<ELFT>::computeContents() {
     addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT,
            sizeof(Elf_Relr));
   }
-  if (part.relrAuthDyn && part.relrAuthDyn->getParent() &&
-      !part.relrAuthDyn->relocs.empty()) {
-    addInSec(DT_AARCH64_AUTH_RELR, *part.relrAuthDyn);
-    addInt(DT_AARCH64_AUTH_RELRSZ, part.relrAuthDyn->getParent()->size);
-    addInt(DT_AARCH64_AUTH_RELRENT, sizeof(Elf_Relr));
-  }
   if (isMain && in.relaPlt->isNeeded()) {
     addInSec(DT_JMPREL, *in.relaPlt);
     entries.emplace_back(DT_PLTRELSZ, addPltRelSz());
@@ -1737,13 +1731,10 @@ template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) {
   }
 }
 
-RelrBaseSection::RelrBaseSection(unsigned concurrency, bool isAArch64Auth)
-    : SyntheticSection(
-          SHF_ALLOC,
-          isAArch64Auth
-              ? SHT_AARCH64_AUTH_RELR
-              : (config->useAndroidRelrTags ? SHT_ANDROID_RELR : SHT_RELR),
-          config->wordsize, isAArch64Auth ? ".relr.auth.dyn" : ".relr.dyn"),
+RelrBaseSection::RelrBaseSection(unsigned concurrency)
+    : SyntheticSection(SHF_ALLOC,
+                       config->useAndroidRelrTags ? SHT_ANDROID_RELR : SHT_RELR,
+                       config->wordsize, ".relr.dyn"),
       relocsVec(concurrency) {}
 
 void RelrBaseSection::mergeRels() {
@@ -2011,8 +2002,8 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() {
 }
 
 template <class ELFT>
-RelrSection<ELFT>::RelrSection(unsigned concurrency, bool isAArch64Auth)
-    : RelrBaseSection(concurrency, isAArch64Auth) {
+RelrSection<ELFT>::RelrSection(unsigned concurrency)
+    : RelrBaseSection(concurrency) {
   this->entsize = config->wordsize;
 }
 
@@ -4783,9 +4774,6 @@ template <class ELFT> void elf::createSyntheticSections() {
     if (config->relrPackDynRelocs) {
       part.relrDyn = std::make_unique<RelrSection<ELFT>>(threadCount);
       add(*part.relrDyn);
-      part.relrAuthDyn = std::make_unique<RelrSection<ELFT>>(
-          threadCount, /*isAArch64Auth=*/true);
-      add(*part.relrAuthDyn);
     }
 
     if (!config->relocatable) {
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index eaa09ea7194fb217303430be53a0bb1d398c7314..34949025a45f7855d84196f358a08e3379b92ce2 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -548,9 +548,7 @@ public:
   static bool classof(const SectionBase *d) {
     return SyntheticSection::classof(d) &&
            (d->type == llvm::ELF::SHT_RELA || d->type == llvm::ELF::SHT_REL ||
-            d->type == llvm::ELF::SHT_RELR ||
-            (d->type == llvm::ELF::SHT_AARCH64_AUTH_RELR &&
-             config->emachine == llvm::ELF::EM_AARCH64));
+            d->type == llvm::ELF::SHT_RELR);
   }
   int32_t dynamicTag, sizeDynamicTag;
   SmallVector<DynamicReloc, 0> relocs;
@@ -598,17 +596,15 @@ private:
 };
 
 struct RelativeReloc {
-  uint64_t getOffset() const {
-    return inputSec->getVA(inputSec->relocs()[relocIdx].offset);
-  }
+  uint64_t getOffset() const { return inputSec->getVA(offsetInSec); }
 
   const InputSectionBase *inputSec;
-  size_t relocIdx;
+  uint64_t offsetInSec;
 };
 
 class RelrBaseSection : public SyntheticSection {
 public:
-  RelrBaseSection(unsigned concurrency, bool isAArch64Auth = false);
+  RelrBaseSection(unsigned concurrency);
   void mergeRels();
   bool isNeeded() const override {
     return !relocs.empty() ||
@@ -626,7 +622,7 @@ template <class ELFT> class RelrSection final : public RelrBaseSection {
   using Elf_Relr = typename ELFT::Relr;
 
 public:
-  RelrSection(unsigned concurrency, bool isAArch64Auth = false);
+  RelrSection(unsigned concurrency);
 
   bool updateAllocSize() override;
   size_t getSize() const override { return relrRelocs.size() * this->entsize; }
@@ -1464,7 +1460,6 @@ struct Partition {
   std::unique_ptr<PackageMetadataNote> packageMetadataNote;
   std::unique_ptr<RelocationBaseSection> relaDyn;
   std::unique_ptr<RelrBaseSection> relrDyn;
-  std::unique_ptr<RelrBaseSection> relrAuthDyn;
   std::unique_ptr<VersionDefinitionSection> verDef;
   std::unique_ptr<SyntheticSection> verNeed;
   std::unique_ptr<VersionTableSection> verSym;
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 0295a656b070533e2769d6093554c84741e9737a..640cb2a445f7dadfd4b6170b90ce338c82363a77 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -783,13 +783,11 @@ template <class ELFT> void Writer<ELFT>::addRelIpltSymbols() {
   // __rela_iplt_{start,end} are initially defined relative to dummy section 0.
   // We'll override Out::elfHeader with relaDyn later when we are sure that
   // .rela.dyn will be present in the output.
-  ElfSym::relaIpltStart = addOptionalRegular(
-      config->isRela ? "__rela_iplt_start" : "__rel_iplt_start",
-      Out::elfHeader, 0, STV_HIDDEN);
-
-  ElfSym::relaIpltEnd = addOptionalRegular(
-      config->isRela ? "__rela_iplt_end" : "__rel_iplt_end",
-      Out::elfHeader, 0, STV_HIDDEN);
+  std::string name = config->isRela ? "__rela_iplt_start" : "__rel_iplt_start";
+  ElfSym::relaIpltStart =
+      addOptionalRegular(name, Out::elfHeader, 0, STV_HIDDEN);
+  name.replace(name.size() - 5, 5, "end");
+  ElfSym::relaIpltEnd = addOptionalRegular(name, Out::elfHeader, 0, STV_HIDDEN);
 }
 
 // This function generates assignments for predefined symbols (e.g. _end or
@@ -921,7 +919,11 @@ static bool shouldSkip(SectionCommand *cmd) {
 static SmallVectorImpl<SectionCommand *>::iterator
 findOrphanPos(SmallVectorImpl<SectionCommand *>::iterator b,
               SmallVectorImpl<SectionCommand *>::iterator e) {
+  // Place non-alloc orphan sections at the end. This matches how we assign file
+  // offsets to non-alloc sections.
   OutputSection *sec = &cast<OutputDesc>(*e)->osec;
+  if (!(sec->flags & SHF_ALLOC))
+    return e;
 
   // As a special case, place .relro_padding before the SymbolAssignment using
   // DATA_SEGMENT_RELRO_END, if present.
@@ -1456,32 +1458,9 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
       in.mipsGot->updateAllocSize();
 
     for (Partition &part : partitions) {
-      // The R_AARCH64_AUTH_RELATIVE has a smaller addend field as bits [63:32]
-      // encode the signing schema. We've put relocations in .relr.auth.dyn
-      // during RelocationScanner::processAux, but the target VA for some of
-      // them might be wider than 32 bits. We can only know the final VA at this
-      // point, so move relocations with large values from .relr.auth.dyn to
-      // .rela.dyn. See also AArch64::relocate.
-      if (part.relrAuthDyn) {
-        auto it = llvm::remove_if(
-            part.relrAuthDyn->relocs, [&part](const RelativeReloc &elem) {
-              const Relocation &reloc = elem.inputSec->relocs()[elem.relocIdx];
-              if (isInt<32>(reloc.sym->getVA(reloc.addend)))
-                return false;
-              part.relaDyn->addReloc({R_AARCH64_AUTH_RELATIVE, elem.inputSec,
-                                      reloc.offset,
-                                      DynamicReloc::AddendOnlyWithTargetVA,
-                                      *reloc.sym, reloc.addend, R_ABS});
-              return true;
-            });
-        changed |= (it != part.relrAuthDyn->relocs.end());
-        part.relrAuthDyn->relocs.erase(it, part.relrAuthDyn->relocs.end());
-      }
       changed |= part.relaDyn->updateAllocSize();
       if (part.relrDyn)
         changed |= part.relrDyn->updateAllocSize();
-      if (part.relrAuthDyn)
-        changed |= part.relrAuthDyn->updateAllocSize();
       if (part.memtagGlobalDescriptors)
         changed |= part.memtagGlobalDescriptors->updateAllocSize();
     }
@@ -1645,14 +1624,6 @@ static void removeUnusedSyntheticSections() {
         auto *sec = cast<SyntheticSection>(s);
         if (sec->getParent() && sec->isNeeded())
           return false;
-        // .relr.auth.dyn relocations may be moved to .rela.dyn in
-        // finalizeAddressDependentContent, making .rela.dyn no longer empty.
-        // Conservatively keep .rela.dyn. .relr.auth.dyn can be made empty, but
-        // we would fail to remove it here.
-        if (config->emachine == EM_AARCH64 && config->relrPackDynRelocs)
-          if (auto *relSec = dyn_cast<RelocationBaseSection>(sec))
-            if (relSec == mainPart->relaDyn.get())
-              return false;
         unused.insert(sec);
         return true;
       });
@@ -1965,10 +1936,6 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
         part.relrDyn->mergeRels();
         finalizeSynthetic(part.relrDyn.get());
       }
-      if (part.relrAuthDyn) {
-        part.relrAuthDyn->mergeRels();
-        finalizeSynthetic(part.relrAuthDyn.get());
-      }
 
       finalizeSynthetic(part.dynSymTab.get());
       finalizeSynthetic(part.gnuHashTab.get());
@@ -2483,11 +2450,12 @@ template <class ELFT> void Writer<ELFT>::assignFileOffsets() {
         lastRX->lastSec == sec)
       off = alignToPowerOf2(off, config->maxPageSize);
   }
-  for (OutputSection *osec : outputSections)
-    if (!(osec->flags & SHF_ALLOC)) {
-      osec->offset = alignToPowerOf2(off, osec->addralign);
-      off = osec->offset + osec->size;
-    }
+  for (OutputSection *osec : outputSections) {
+    if (osec->flags & SHF_ALLOC)
+      continue;
+    osec->offset = alignToPowerOf2(off, osec->addralign);
+    off = osec->offset + osec->size;
+  }
 
   sectionHeaderOff = alignToPowerOf2(off, config->wordsize);
   fileSize = sectionHeaderOff + (outputSections.size() + 1) * sizeof(Elf_Shdr);
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index e7a913e025da966ebaaf704a446e564445d46f19..79ddc159a945f704c6f061dd893db0cabb38de3e 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -44,6 +44,10 @@ ELF Improvements
   (typical for embedded). It also makes full LTO feasible in such cases, since
   IR merging currently prevents the linker script from referring to input
   files. (`#90007 <https://github.com/llvm/llvm-project/pull/90007>`_)
+* ``--force-group-allocation`` is implemented to discard ``SHT_GROUP`` sections
+  and combine relocation sections if their relocated section group members are
+  placed to the same output section.
+  (`#94704 <https://github.com/llvm/llvm-project/pull/94704>`_)
 
 Breaking changes
 ----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index da3b926d02a28bdee21d9e3a1105b970dadaab86..bdc35c08ccb8e22b32dc255620663cc302453967 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -278,6 +278,8 @@ Set the
 field to the specified value.
 .It Fl -fini Ns = Ns Ar symbol
 Specify a finalizer function.
+.It Fl -force-group-allocation
+Only meaningful for -r. Section groups are discarded. If two section group members are placed to the same output section, combine their relocations as well.
 .It Fl -format Ns = Ns Ar input-format , Fl b Ar input-format
 Specify the format of the inputs following this option.
 .Ar input-format
diff --git a/lld/test/ELF/aarch64-reloc-pauth.s b/lld/test/ELF/aarch64-reloc-pauth.s
index f1ce29eaae4679b69576368fabc64067cf717b85..0cfcb1665b93972886e9a00d9f8441ed8df49b2c 100644
--- a/lld/test/ELF/aarch64-reloc-pauth.s
+++ b/lld/test/ELF/aarch64-reloc-pauth.s
@@ -1,13 +1,12 @@
 # REQUIRES: aarch64
 
-# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: rm -rf %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %p/Inputs/shared2.s -o %t.a.o
+# RUN: ld.lld -shared %t.a.o -soname=so -o %t.a.so
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
 
-# RUN: llvm-mc -filetype=obj -triple=aarch64 %p/Inputs/shared2.s -o a.o
-# RUN: ld.lld -shared a.o -soname=so -o a.so
-# RUN: llvm-mc -filetype=obj -triple=aarch64 main.s -o main.o
-
-# RUN: ld.lld -pie main.o a.so -o main
-# RUN: llvm-readobj -r main | FileCheck --check-prefix=UNPACKED %s
+# RUN: ld.lld -pie %t.o %t.a.so -o %t
+# RUN: llvm-readobj -r %t | FileCheck --check-prefix=UNPACKED %s
 
 # UNPACKED:          Section ({{.+}}) .rela.dyn {
 # UNPACKED-NEXT:       0x30470 R_AARCH64_AUTH_RELATIVE - 0x1
@@ -23,8 +22,8 @@
 # UNPACKED-NEXT:       0x304B0 R_AARCH64_AUTH_ABS64 bar2 0x0
 # UNPACKED-NEXT:     }
 
-# RUN: ld.lld main.o a.so -o main.nopie
-# RUN: llvm-readobj -r main.nopie | FileCheck --check-prefix=NOPIE %s
+# RUN: ld.lld %t.o %t.a.so -o %t.nopie
+# RUN: llvm-readobj -r %t.nopie | FileCheck --check-prefix=NOPIE %s
 
 # NOPIE:      Section ({{.+}}) .rela.dyn {
 # NOPIE:        0x230460 R_AARCH64_AUTH_RELATIVE - 0x200001
@@ -40,95 +39,67 @@
 # NOPIE-NEXT:   0x2304A0 R_AARCH64_AUTH_ABS64 bar2 0x0
 # NOPIE-NEXT: }
 
-# RUN: ld.lld -pie -z pack-relative-relocs main.o a.so -o main.pie
-# RUN: llvm-readelf -S -d -r -x .test main.pie | FileCheck --check-prefixes=RELR,HEX %s
-
-# RELR:      Section Headers:
-# RELR-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
-# RELR:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000090 18 A 1 0 8
-# RELR:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000018 08 A 0 0 8
-
-# RELR:      Dynamic section at offset {{.+}} contains 16 entries
-# RELR:      0x0000000070000012 (AARCH64_AUTH_RELR) 0x[[ADDR2]]
-# RELR-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 24 (bytes)
-# RELR-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 8 (bytes)
-
-## Decoded SHT_RELR section is same as UNPACKED,
-## but contains only the relative relocations.
-## Any relative relocations with odd offset or value wider than 32 bits stay in SHT_RELA.
-
-# RELR:      Relocation section '.rela.dyn' at offset 0x[[ADDR1]] contains 6 entries:
-# RELR-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
-# RELR-NEXT: 0000000000030460  0000000000000411 R_AARCH64_AUTH_RELATIVE           123456789a
-# RELR-NEXT: 0000000000030468  0000000000000411 R_AARCH64_AUTH_RELATIVE           ffffffedcba98766
-# RELR-NEXT: 0000000000030470  0000000000000411 R_AARCH64_AUTH_RELATIVE           8003043f
-# RELR-NEXT: 0000000000030489  0000000000000411 R_AARCH64_AUTH_RELATIVE           4
-# RELR-NEXT: 0000000000030478  0000000100000244 R_AARCH64_AUTH_ABS64   0000000000000000 zed2 + 1111
-# RELR-NEXT: 0000000000030480  0000000200000244 R_AARCH64_AUTH_ABS64   0000000000000000 bar2 + 0
-# RELR-EMPTY:
-# RELR-NEXT: Relocation section '.relr.auth.dyn' at offset 0x[[ADDR2]] contains 5 entries:
-# RELR-NEXT: Index: Entry Address Symbolic Address
-# RELR-NEXT: 0000: 0000000000030440 0000000000030440 $d.0
-# RELR-NEXT: 0001: 000000000000000f 0000000000030448 $d.0 + 0x8
-# RELR-NEXT:  0000000000030450 $d.0 + 0x10
-# RELR-NEXT:  0000000000030458 $d.0 + 0x18
-# RELR-NEXT: 0002: 0000000000030492 0000000000030492 $d.0 + 0x52
+# RUN: ld.lld -pie %t.o %t.a.so -o %t.pie
+# RUN: llvm-readelf -S -d -r -x .test %t.pie | FileCheck --check-prefixes=PIE,HEX %s
+
+# PIE:      Section Headers:
+# PIE-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
+# PIE:      .rela.dyn RELA {{0*}}[[#%x,ADDR1:]]
+# PIE-SAME:                                     {{0*}}[[#ADDR1]] 000108 18 A 1 0 8
+
+# PIE:      Relocation section '.rela.dyn' at offset 0x[[#ADDR1]] contains 11 entries:
+# PIE-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# PIE-NEXT: 0000000000030470  0000000000000411 R_AARCH64_AUTH_RELATIVE 1
+# PIE-NEXT: 0000000000030478  0000000000000411 R_AARCH64_AUTH_RELATIVE 30472
+# PIE-NEXT: 0000000000030480  0000000000000411 R_AARCH64_AUTH_RELATIVE fffffffffffffffd
+# PIE-NEXT: 0000000000030488  0000000000000411 R_AARCH64_AUTH_RELATIVE 12345678
+# PIE-NEXT: 0000000000030490  0000000000000411 R_AARCH64_AUTH_RELATIVE 123456789a
+# PIE-NEXT: 0000000000030498  0000000000000411 R_AARCH64_AUTH_RELATIVE ffffffedcba98766
+# PIE-NEXT: 00000000000304a0  0000000000000411 R_AARCH64_AUTH_RELATIVE 8003046f
+# PIE-NEXT: 00000000000304b9  0000000000000411 R_AARCH64_AUTH_RELATIVE 4
+# PIE-NEXT: 00000000000304c2  0000000000000411 R_AARCH64_AUTH_RELATIVE 30475
+# PIE-NEXT: 00000000000304a8  0000000100000244 R_AARCH64_AUTH_ABS64   0000000000000000 zed2 + 1111
+# PIE-NEXT: 00000000000304b0  0000000200000244 R_AARCH64_AUTH_ABS64   0000000000000000 bar2 + 0
 
 # HEX:      Hex dump of section '.test':
-# HEX-NEXT: 0x00030440 01000000 2a000020 42040300 2b000000
-##                     ^^^^^^^^ Implicit val = 1 = __ehdr_start + 1
+# HEX-NEXT: 0x00030470 00000000 2a000020 00000000 2b000000
 ##                              ^^^^ Discr = 42
 ##                                    ^^ Key (bits 5..6) = DA
-##                                       ^^^^^^^^ Implicit val = 0x30442 = 0x30440 + 2 = .test + 2
 ##                                                ^^^^ Discr = 43
 ##                                                      ^^ Key (bits 5..6) = IA
-# HEX-NEXT: 0x00030450 fdffffff 2c000080 78563412 2d000020
-##                     ^^^^^^^^ Implicit val = -3 = __ehdr_start - 3
+# HEX-NEXT: 0x00030480 00000000 2c000080 00000000 2d000020
 ##                              ^^^^ Discr = 44
 ##                                    ^^ Key (bits 5..6) = IA
 ##                                    ^^ Addr diversity (bit 7) = true
-##                                       ^^^^^^^^ Implicit val = 0x12345678 = __ehdr_start + 0x12345678
 ##                                                ^^^^ Discr = 45
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x00030460 00000000 2e000020 00000000 2f000020
-##                     ^^^^^^^^ No implicit val (rela reloc due val wider than 32 bits)
+# HEX-NEXT: 0x00030490 00000000 2e000020 00000000 2f000020
 ##                              ^^^^ Discr = 46
 ##                                    ^^ Key (bits 5..6) = DA
-##                                       ^^^^^^^^ No implicit val (rela reloc due to val wider than 32 bits)
 ##                                                ^^^^ Discr = 47
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x00030470 00000000 30000020 00000000 31000020
-##                     ^^^^^^^^ No implicit val (rela reloc due val wider than 32 bits)
+# HEX-NEXT: 0x000304a0 00000000 30000020 00000000 31000020
 ##                              ^^^^ Discr = 48
 ##                                    ^^ Key (bits 5..6) = DA
-##                                       ^^^^^^^^ No implicit val (rela reloc due to a preemptible symbol)
 ##                                                ^^^^ Discr = 49
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x00030480 00000000 32000000 77000000 00330000
-##                     ^^^^^^^^ No implicit val (rela reloc due to a preemptible symbol)
+# HEX-NEXT: 0x000304b0 00000000 32000000 77000000 00330000
 ##                              ^^^^ Discr = 50
 ##                                    ^^ Key (bits 5..6) = IA
-##                                         ^^^^^^ ^^ No implicit val (rela reloc due to odd offset)
 ##                                                  ^^^^ Discr = 51
-# HEX-NEXT: 0x00030490 20774504 03003400 0020{{\ }}
+# HEX-NEXT: 0x000304c0 20770000 00003400 0020{{\ }}
 ##                     ^^ Key (bits 5..6) = DA
-##                         ^^^^ ^^^^ Implicit val = 0x30445 = 0x30440 + 5 = .test + 5
 ##                                  ^^^^ Discr = 52
 ##                                         ^^ Key (bits 5..6) = DA
 
-#--- main.s
-
 .section .test, "aw"
 .p2align 3
 .quad (__ehdr_start + 1)@AUTH(da,42)
 .quad (.test + 2)@AUTH(ia,43)
 .quad (__ehdr_start - 3)@AUTH(ia,44,addr)
 .quad (__ehdr_start + 0x12345678)@AUTH(da,45)
-## Addend wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (__ehdr_start + 0x123456789A)@AUTH(da,46)
-## Negative addend wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (__ehdr_start - 0x123456789A)@AUTH(da,47)
-## INT32_MAX plus non-zero .test is wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (.test + 0x7FFFFFFF)@AUTH(da,48)
 .quad (zed2 + 0x1111)@AUTH(da,49)
 .quad bar2@AUTH(ia,50)
@@ -136,71 +107,3 @@
 .quad (__ehdr_start + 4)@AUTH(da,51)
 .byte 0x77
 .quad (.test + 5)@AUTH(da,52)
-
-#--- empty-relr.s
-
-## .relr.auth.dyn relocations that do not fit 32 bits are moved to .rela.dyn.
-## In this case .relr.auth.dyn will be made empty, but
-## removeUnusedSyntheticSections fails to remove the section.
-
-# RUN: llvm-mc -filetype=obj -triple=aarch64 empty-relr.s -o empty-relr.o
-# RUN: ld.lld -pie -z pack-relative-relocs empty-relr.o -o empty-relr
-# RUN: llvm-readelf -S -d -r empty-relr | FileCheck --check-prefixes=EMPTY-RELR %s
-
-# EMPTY-RELR:      Section Headers:
-# EMPTY-RELR-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
-# EMPTY-RELR:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000018 18 A 0 0 8
-# EMPTY-RELR:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000000 08 A 0 0 8
-
-# EMPTY-RELR:      Dynamic section at offset {{.+}} contains 12 entries
-# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELR)
-# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELRSZ)
-# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELRENT)
-# EMPTY-RELR:      0x0000000000000007 (RELA) 0x[[ADDR1]]
-# EMPTY-RELR-NEXT: 0x0000000000000008 (RELASZ) 24 (bytes)
-# EMPTY-RELR-NEXT: 0x0000000000000009 (RELAENT) 24 (bytes)
-
-# EMPTY-RELR:      Relocation section '.rela.dyn' at offset {{.+}} contains 1 entries:
-# EMPTY-RELR-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
-# EMPTY-RELR-NEXT: 0000000000030320  0000000000000411 R_AARCH64_AUTH_RELATIVE           8003031f
-# EMPTY-RELR-EMPTY:
-# EMPTY-RELR-NEXT: Relocation section '.relr.auth.dyn' at offset {{.+}} contains 0 entries:
-# EMPTY-RELR-NEXT: Index: Entry Address Symbolic Address
-
-.section .test, "aw"
-.p2align 3
-.quad (.test + 0x7FFFFFFF)@AUTH(da,42)
-
-#--- empty-rela.s
-
-## .relr.auth.dyn relocations that do not fit 32 bits are moved to .rela.dyn.
-## If this scenario does not happen, .rela.dyn will remain empty,
-## but removeUnusedSyntheticSections fails to remove the section.
-
-# RUN: llvm-mc -filetype=obj -triple=aarch64 empty-rela.s -o empty-rela.o
-# RUN: ld.lld -pie -z pack-relative-relocs empty-rela.o -o empty-rela
-# RUN: llvm-readelf -S -d -r empty-rela | FileCheck --check-prefixes=EMPTY-RELA %s
-
-# EMPTY-RELA:      Section Headers:
-# EMPTY-RELA-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
-# EMPTY-RELA:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000000 18 A 0 0 8
-# EMPTY-RELA:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000008 08 A 0 0 8
-
-# EMPTY-RELA:      Dynamic section at offset {{.+}} contains 12 entries
-# EMPTY-RELA-NOT:  (RELR)
-# EMPTY-RELA-NOT:  (RELRSZ)
-# EMPTY-RELA-NOT:  (RELRENT)
-# EMPTY-RELA:      0x0000000070000012 (AARCH64_AUTH_RELR) 0x[[ADDR2]]
-# EMPTY-RELA-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 8 (bytes)
-# EMPTY-RELA-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 8 (bytes)
-
-# EMPTY-RELA:      Relocation section '.rela.dyn' at offset {{.+}} contains 0 entries:
-# EMPTY-RELA-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name
-# EMPTY-RELA-EMPTY:
-# EMPTY-RELA-NEXT: Relocation section '.relr.auth.dyn' at offset {{.+}} contains 1 entries:
-# EMPTY-RELA-NEXT: Index: Entry Address Symbolic Address
-# EMPTY-RELA-NEXT: 0000: 0000000000030310 0000000000030310 $d.0
-
-.section .test, "aw"
-.p2align 3
-.quad (.test + 0x12345678)@AUTH(da,42)
diff --git a/lld/test/ELF/comdat.s b/lld/test/ELF/comdat.s
index a9a5f5fa3bb6217a0cd292e155f66584c8471248..32dfc91cf5509149b265f7a0006c1f9359c6438d 100644
--- a/lld/test/ELF/comdat.s
+++ b/lld/test/ELF/comdat.s
@@ -4,6 +4,7 @@
 // RUN: ld.lld -shared %t.o %t2.o -o %t
 // RUN: llvm-objdump -d %t | FileCheck %s
 // RUN: llvm-readelf -S -s %t | FileCheck --check-prefix=READ %s
+// RUN: ld.lld -shared --force-group-allocation %t.o %t2.o -o - | cmp - %t
 
 // Check that we don't crash with --gc-section and that we print a list of
 // reclaimed sections on stderr.
diff --git a/lld/test/ELF/fatlto/fatlto.test b/lld/test/ELF/fatlto/fatlto.test
index ed137087746e9d38a01714a62a7166f357ecab97..7ec094d935cc5e9a1006e2bdbfc5171039361224 100644
--- a/lld/test/ELF/fatlto/fatlto.test
+++ b/lld/test/ELF/fatlto/fatlto.test
@@ -8,7 +8,6 @@
 ; RUN: opt < a-LTO.ll --module-summary -o a-fatLTO.bc
 ; RUN: llvm-objcopy --add-section=.llvm.lto=a-fatLTO.bc --set-section-flags=.llvm.lto=exclude --set-section-type=.llvm.lto=0x6fff4c0c a-fatLTO.o
 
-
 ; RUN: llc main-LTO.ll --filetype=obj -o main-fatLTO.o --relocation-model=pic
 ; RUN: opt < main-LTO.ll --module-summary -o main-fatLTO.bc
 ; RUN: llvm-objcopy --add-section=.llvm.lto=main-fatLTO.bc --set-section-flags=.llvm.lto=exclude --set-section-type=.llvm.lto=0x6fff4c0c main-fatLTO.o
@@ -17,11 +16,6 @@
 ; RUN: llvm-readelf -S main-fatLTO.o | FileCheck --check-prefix=HAS_LLVM_LTO %s
 
 ;; Make sure that the section flags are set correctly
-; HA_LLVM_LTO: Name: .llvm.lto
-; HA_LLVM_LTO-NEXT: Type: SHT_LLVM_LTO
-; HA_LLVM_LTO-NEXT: Flags
-; HA_LLVM_LTO-NEXT: SHF_EXCLUDE
-
 ; HAS_LLVM_LTO: Name Type Address Off Size ES Flg Lk Inf Al
 ; HAS_LLVM_LTO: .llvm.lto LLVM_LTO {{.*}} 00  WE  0   0  1
 
@@ -64,16 +58,13 @@
 ; RUN: ld.lld -o foo-fatLTO.archive a.a main-LTO.bc --fat-lto-objects
 ; RUN: cmp foo-fatLTO.archive foo-LTO
 
-;; Test FatLTO works with relocatable links using PIC objects
-;; Currently, with PIC relocatable links, FatLTO sections are treated as
-;; orphan sections and incorrectly concatenated together. This test verifies
-;; the current behavior, but should be fixed to either merge those sections
-;; correctly, or to drop them altogether.
+;; Test FatLTO works with relocatable links using PIC objects, and that
+;; SHT_LLVM_LTO sections are discarded.
 ; RUN: llvm-ar rcs fatLTO-pic.a a-fatLTO.o main-fatLTO.o
 ; RUN: llvm-readelf -S fatLTO-pic.a | FileCheck --check-prefix=HAS_LLVM_LTO %s
 
-; RUN: ld.lld --whole-archive fatLTO-pic.a -r -o fatLTO-pic-reolcatable.o
-; RUN: llvm-readelf -S fatLTO-pic-reolcatable.o | FileCheck --check-prefix=HAS_LLVM_LTO %s
+; RUN: ld.lld --whole-archive fatLTO-pic.a -r -o fatLTO-pic-relocatable.o
+; RUN: llvm-readelf -S fatLTO-pic-relocatable.o | FileCheck --check-prefix=CHECK-NON-LTO-TARGET %s
 
 ;--- a-LTO.ll
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test b/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test
index 2dcd0f8d6ce2fa2a4239332a72f7cf2f509811ee..eabdf75fcf935f7f335fdfa3b1e79057d81de72d 100644
--- a/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test
+++ b/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test
@@ -16,20 +16,20 @@
 ## The output file must include all sections.
 # RUN: llvm-readelf -S %t/a.elf | FileCheck %s
 
-# CHECK:      There are 12 section headers, starting at offset 0x2140:
+# CHECK:      There are 12 section headers, starting at offset 0x2138:
 # CHECK:      [Nr] Name         Type      Address          Off    Size   ES Flg Lk Inf Al
 # CHECK-NEXT: [ 0]              NULL      0000000000000000 000000 000000 00     0  0   0
 # CHECK-NEXT: [ 1] .nonalloc    PROGBITS  0000000000000000 001064 001000 00 W   0  0   1
-# CHECK-NEXT: [ 2] .comment     PROGBITS  0000000000000000 {{.*}} {{.*}} 01 MS  0  0   1
-# CHECK-NEXT: [ 3] .symtab      SYMTAB    0000000000000000 {{.*}} {{.*}} 18     5  1   8
-# CHECK-NEXT: [ 4] .shstrtab    STRTAB    0000000000000000 {{.*}} {{.*}} 00     0  0   1
-# CHECK-NEXT: [ 5] .strtab      STRTAB    0000000000000000 {{.*}} {{.*}} 00     0  0   1
-# CHECK-NEXT: [ 6] .dat         PROGBITS  0000000000000000 002137 000004 00 W   0  0   1
-# CHECK-NEXT: [ 7] .intvec0_out PROGBITS  0000000000000000 00213b 000000 00 W   0  0   1
-# CHECK-NEXT: [ 8] .intvec1_out PROGBITS  0000000000000000 00213b 000000 00 W   0  0   1
-# CHECK-NEXT: [ 9] .intvec2_out PROGBITS  0000000000000000 00213b 000000 00 W   0  0   1
-# CHECK-NEXT: [10] .intvec3_out PROGBITS  00000000803fe060 001060 000004 00 AX  0  0   1
-# CHECK-NEXT: [11] .text        PROGBITS  00000000803fe064 001064 000000 00 AX  0  0   4
+# CHECK-NEXT: [ 2] .dat         PROGBITS  0000000000000000 002064 000004 00 W   0  0   1
+# CHECK-NEXT: [ 3] .intvec0_out PROGBITS  0000000000000000 002068 000000 00 W   0  0   1
+# CHECK-NEXT: [ 4] .intvec1_out PROGBITS  0000000000000000 002068 000000 00 W   0  0   1
+# CHECK-NEXT: [ 5] .intvec2_out PROGBITS  0000000000000000 002068 000000 00 W   0  0   1
+# CHECK-NEXT: [ 6] .intvec3_out PROGBITS  00000000803fe060 001060 000004 00 AX  0  0   1
+# CHECK-NEXT: [ 7] .text        PROGBITS  00000000803fe064 001064 000000 00 AX  0  0   4
+# CHECK-NEXT: [ 8] .comment     PROGBITS  0000000000000000 {{.*}} {{.*}} 01 MS  0  0   1
+# CHECK-NEXT: [ 9] .symtab      SYMTAB    0000000000000000 {{.*}} {{.*}} 18     11 1   8
+# CHECK-NEXT: [10] .shstrtab    STRTAB    0000000000000000 {{.*}} {{.*}} 00     0  0   1
+# CHECK-NEXT: [11] .strtab      STRTAB    0000000000000000 {{.*}} {{.*}} 00     0  0   1
 
 
 #--- a.s
diff --git a/lld/test/ELF/linkerscript/sections-nonalloc.s b/lld/test/ELF/linkerscript/sections-nonalloc.s
index 79765d32dfffedc834c05278e175876117632025..d66e524248ceefa4710514465124dd33cfcb09d9 100644
--- a/lld/test/ELF/linkerscript/sections-nonalloc.s
+++ b/lld/test/ELF/linkerscript/sections-nonalloc.s
@@ -16,15 +16,15 @@
 # CHECK-NEXT:  [ 2] data1     PROGBITS 0000000000000001 001001 000001 00  WA  0
 # CHECK-NEXT:  [ 3] other1    PROGBITS 0000000000000000 001008 000001 00      0
 # CHECK-NEXT:  [ 4] other2    PROGBITS 0000000000000000 001010 000001 00      0
-## Orphan placement places other3, .symtab, .shstrtab and .strtab after other2.
-# CHECK-NEXT:  [ 5] other3    PROGBITS 0000000000000000 001020 000001 00      0
-# CHECK-NEXT:  [ 6] .symtab   SYMTAB   0000000000000000 001028 000030 18      8
-# CHECK-NEXT:  [ 7] .shstrtab STRTAB   0000000000000000 001058 00004d 00      0
-# CHECK-NEXT:  [ 8] .strtab   STRTAB   0000000000000000 0010a5 000008 00      0
-# CHECK-NEXT:  [ 9] data2     PROGBITS 0000000000000002 001002 000001 00  WA  0
+# CHECK-NEXT:  [ 5] data2     PROGBITS 0000000000000002 001002 000001 00  WA  0
 ## max{sortRank(data1),sortRank(data2)} <= sortRank(data3). data3 is placed after the latter.
-# CHECK-NEXT:  [10] data3     PROGBITS 0000000000000003 001003 000001 00  WA  0
-# CHECK-NEXT:  [11] .text     PROGBITS 0000000000000004 001004 000001 00  AX  0
+# CHECK-NEXT:  [ 6] data3     PROGBITS 0000000000000003 001003 000001 00  WA  0
+# CHECK-NEXT:  [ 7] .text     PROGBITS 0000000000000004 001004 000001 00  AX  0
+## Non-alloc orphan sections other3, .symtab, .shstrtab and .strtab are placed at the end.
+# CHECK-NEXT:  [ 8] other3    PROGBITS 0000000000000000 001020 000001 00      0
+# CHECK-NEXT:  [ 9] .symtab   SYMTAB   0000000000000000 001028 000030 18     11
+# CHECK-NEXT:  [10] .shstrtab STRTAB   0000000000000000 001058 00004d 00      0
+# CHECK-NEXT:  [11] .strtab   STRTAB   0000000000000000 0010a5 000008 00      0
 
 # CHECK:       Type       Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
 # CHECK-NEXT:  LOAD       0x001000 0x0000000000000000 0x0000000000000000 0x000004 0x000004 RW  0x1000
@@ -34,6 +34,11 @@
 # RUN: ld.lld -T %t/b.lds %t.o -o %tb
 # RUN: llvm-readelf -S -l %tb | FileCheck %s --check-prefix=CHECK1
 
+## --section-start causes the orphan other3 to be considered before .data3.
+## The non-alloc other3 does not disable the placement of .data3.
+# RUN: ld.lld -T %t/b.lds %t.o -o %tb --section-start=other3=0
+# RUN: llvm-readelf -S -l %tb | FileCheck %s --check-prefix=CHECK1
+
 # CHECK1:      [Nr] Name      Type     Address          Off    Size   ES Flg Lk
 # CHECK1-NEXT: [ 0]           NULL     0000000000000000 000000 000000 00      0
 # CHECK1-NEXT: [ 1] .text     PROGBITS 00000000000000b0 0000b0 000001 00  AX  0
@@ -41,12 +46,12 @@
 # CHECK1-NEXT: [ 3] data1     PROGBITS 00000000000000b2 0000b2 000001 00  WA  0
 # CHECK1-NEXT: [ 4] other1    PROGBITS 0000000000000000 0000b8 000001 00      0
 # CHECK1-NEXT: [ 5] other2    PROGBITS 0000000000000000 0000c0 000001 00      0
-# CHECK1-NEXT: [ 6] other3    PROGBITS 0000000000000000 0000d0 000001 00      0
-# CHECK1-NEXT: [ 7] .symtab   SYMTAB   0000000000000000 0000d8 000030 18      9
-# CHECK1-NEXT: [ 8] .shstrtab STRTAB   0000000000000000 000108 00004d 00      0
-# CHECK1-NEXT: [ 9] .strtab   STRTAB   0000000000000000 000155 000008 00      0
-# CHECK1-NEXT: [10] data2     PROGBITS 00000000000000b3 0000b3 000001 00  WA  0
-# CHECK1-NEXT: [11] data3     PROGBITS 00000000000000b4 0000b4 000001 00  WA  0
+# CHECK1-NEXT: [ 6] data2     PROGBITS 00000000000000b3 0000b3 000001 00  WA  0
+# CHECK1-NEXT: [ 7] data3     PROGBITS 00000000000000b4 0000b4 000001 00  WA  0
+# CHECK1-NEXT: [ 8] other3    PROGBITS 0000000000000000 0000d0 000001 00      0
+# CHECK1-NEXT: [ 9] .symtab   SYMTAB   0000000000000000 0000d8 000030 18     11
+# CHECK1-NEXT: [10] .shstrtab STRTAB   0000000000000000 000108 00004d 00      0
+# CHECK1-NEXT: [11] .strtab   STRTAB   0000000000000000 000155 000008 00      0
 # CHECK1:      Type       Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
 # CHECK1-NEXT: LOAD       0x000000 0x0000000000000000 0x0000000000000000 0x0000b5 0x0000b5 RWE 0x1000
 # CHECK1-NEXT: 0x60000000 0x0000b8 0x0000000000000000 0x0000000000000000 0x000009 0x000001     0x8
diff --git a/lld/test/ELF/linkerscript/sections.s b/lld/test/ELF/linkerscript/sections.s
index 5d6cc1f3bd0df88f32b34dea44000560f5654fe8..fc03af8402dfe3f9887c4c309adb3e3726594a1d 100644
--- a/lld/test/ELF/linkerscript/sections.s
+++ b/lld/test/ELF/linkerscript/sections.s
@@ -79,8 +79,8 @@
 # SEP-BY-NONALLOC:      [ 1] .text     PROGBITS 0000000000000000 001000 00000e 00  AX
 # SEP-BY-NONALLOC-NEXT: [ 2] .data     PROGBITS 000000000000000e 00100e 000020 00  WA
 # SEP-BY-NONALLOC-NEXT: [ 3] .comment  PROGBITS 0000000000000000 001031 000008 01  MS
-# SEP-BY-NONALLOC:      [ 7] other     PROGBITS 000000000000002e 00102e 000003 00  WA
-# SEP-BY-NONALLOC-NEXT: [ 8] .bss      NOBITS   0000000000000031 001031 000002 00  WA
+# SEP-BY-NONALLOC:      [ 4] other     PROGBITS 000000000000002e 00102e 000003 00  WA
+# SEP-BY-NONALLOC-NEXT: [ 5] .bss      NOBITS   0000000000000031 001031 000002 00  WA
 
 # SEP-BY-NONALLOC:      Type      Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
 # SEP-BY-NONALLOC-NEXT: LOAD      0x001000 0x0000000000000000 0x0000000000000000 0x00000e 0x00000e R E 0x1000
diff --git a/lld/test/ELF/relocatable-comdat.s b/lld/test/ELF/relocatable-comdat.s
index 160c48d9fb4d9011c053538668b384342fdee6f0..c69e4928509d4384e9d3400d6c306dd3ed79927b 100644
--- a/lld/test/ELF/relocatable-comdat.s
+++ b/lld/test/ELF/relocatable-comdat.s
@@ -3,46 +3,81 @@
 ## may be rewritten because group members may change their indices. Additionally,
 ## group member may be combined or discarded (e.g. /DISCARD/ or --gc-sections).
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: ld.lld -r %t.o %t.o -o %t.ro
-# RUN: llvm-readelf -g -S %t.ro | FileCheck %s
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: ld.lld -r a.o a.o -o a.ro
+# RUN: llvm-readelf -g -S a.ro | FileCheck %s
 
-# CHECK: Name    Type  Address          Off    Size   ES Flg Lk        Inf   Al
-# CHECK: .group  GROUP 0000000000000000 {{.*}} 000014 04     {{[1-9]}} [[#]]  4
+# CHECK:      Name           Type     Address          Off    Size   ES Flg Lk    Inf   Al
+# CHECK:      .group         GROUP    0000000000000000 {{.*}} 00001c 04     [[#]] [[#]]  4
+# CHECK-NEXT: .rodata.bar    PROGBITS 0000000000000000 {{.*}} 000001 00  AG  0      0    1
+# CHECK-NEXT: .rodata.foo    PROGBITS 0000000000000000 {{.*}} 000001 00  AG  0      0    1
+# CHECK-NEXT: .text.bar      PROGBITS 0000000000000000 {{.*}} 000008 00 AXG  0      0    1
+# CHECK-NEXT: .rela.text.bar RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# CHECK-NEXT: .text.foo      PROGBITS 0000000000000000 {{.*}} 000008 00 AXG [[#]] [[#]]  1
+# CHECK-NEXT: .rela.text.foo RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# CHECK-NEXT: .note.GNU-stack
 
-# CHECK: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections:
+# CHECK: COMDAT group section [{{.*}}] `.group' [abc] contains 6 sections:
 # CHECK-NEXT: Name
 # CHECK-NEXT: .rodata.bar
 # CHECK-NEXT: .rodata.foo
 # CHECK-NEXT: .text.bar
+# CHECK-NEXT: .rela.text.bar
 # CHECK-NEXT: .text.foo
+# CHECK-NEXT: .rela.text.foo
 
 ## Rewrite SHT_GROUP content if some members are combined.
-# RUN: echo 'SECTIONS { .rodata : {*(.rodata.*)} .text : {*(.text.*)} }' > %t1.lds
-# RUN: ld.lld -r -T %t1.lds %t.o %t.o -o %t1.ro
-# RUN: llvm-readelf -g -S %t1.ro | FileCheck %s --check-prefix=SCRIPT1
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.*)} .text : {*(.text.*)} }' > combine.lds
+# RUN: ld.lld -r -T combine.lds a.o a.o -o combine.ro
+# RUN: llvm-readelf -g -S combine.ro | FileCheck %s --check-prefix=COMBINE
 
-# SCRIPT1: Name    Type  Address          Off    Size   ES Flg Lk        Inf   Al
-# SCRIPT1: .group  GROUP 0000000000000000 {{.*}} 00000c 04     {{[1-9]}} [[#]]  4
+# COMBINE:      Name           Type     Address          Off    Size   ES Flg Lk    Inf   Al
+# COMBINE:      .rodata        PROGBITS 0000000000000000 {{.*}} 000002 00  AG  0      0    1
+# COMBINE-NEXT: .text          PROGBITS 0000000000000000 {{.*}} 000010 00 AXG  0      0    4
+# COMBINE-NEXT: .group         GROUP    0000000000000000 {{.*}} 000014 04     [[#]] [[#]]  4
+# COMBINE-NEXT: .rela.text     RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# COMBINE-NEXT: .rela.text     RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# COMBINE-NEXT: .note.GNU-stack
 
-# SCRIPT1: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections:
-# SCRIPT1-NEXT: Name
-# SCRIPT1-NEXT: .rodata
-# SCRIPT1-NEXT: .text
+# COMBINE: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections:
+# COMBINE-NEXT: Name
+# COMBINE-NEXT: .rodata
+# COMBINE-NEXT: .text
+# COMBINE-NEXT: .rela.text
+# COMBINE-NEXT: .rela.text
 
-# RUN: echo 'SECTIONS { /DISCARD/ : {*(.rodata.*)} }' > %t2.lds
-# RUN: ld.lld -r -T %t2.lds %t.o %t.o -o %t2.ro
-# RUN: llvm-readelf -g -S %t2.ro | FileCheck %s --check-prefix=SCRIPT2
+## If --force-group-allocation is specified, discard .group and combine .rela.* if their relocated sections are combined.
+# RUN: ld.lld -r -T combine.lds a.o a.o --force-group-allocation -o combine-a.ro
+# RUN: llvm-readelf -g -S combine-a.ro | FileCheck %s --check-prefix=COMBINE-A
+
+# COMBINE-A:      Name            Type     Address          Off    Size   ES Flg Lk    Inf   Al
+# COMBINE-A:      .rodata         PROGBITS 0000000000000000 {{.*}} 000002 00   A  0      0    1
+# COMBINE-A-NEXT: .text           PROGBITS 0000000000000000 {{.*}} 000010 00  AX  0      0    4
+# COMBINE-A-NEXT: .rela.text      RELA     0000000000000000 {{.*}} 000030 18   I [[#]] [[#]]  8
+# COMBINE-A-NEXT: .note.GNU-stack
+
+# RUN: echo 'SECTIONS { /DISCARD/ : {*(.rodata.*)} }' > discard-rodata.lds
+# RUN: ld.lld -r -T discard-rodata.lds a.o a.o -o discard-rodata.ro
+# RUN: llvm-readelf -g -S discard-rodata.ro | FileCheck %s --check-prefix=NO-RODATA
 
 ## Handle discarded group members.
-# SCRIPT2: [Nr] Name    Type  Address          Off    Size   ES Flg Lk        Inf   Al
-# SCRIPT2: [ 2] .group  GROUP 0000000000000000 {{.*}} 00000c 04     {{[1-9]}} [[#]]  4
+# NO-RODATA:      Name           Type     Address          Off    Size   ES Flg Lk    Inf   Al
+# NO-RODATA:      .group         GROUP    0000000000000000 {{.*}} 000014 04     [[#]] [[#]]  4
+# NO-RODATA-NEXT: .text.bar      PROGBITS 0000000000000000 {{.*}} 000008 00 AXG  0      0    1
+# NO-RODATA-NEXT: .rela.text.bar RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# NO-RODATA-NEXT: .text.foo      PROGBITS 0000000000000000 {{.*}} 000008 00 AXG [[#]] [[#]]  1
+# NO-RODATA-NEXT: .rela.text.foo RELA     0000000000000000 {{.*}} 000018 18  IG [[#]] [[#]]  8
+# NO-RODATA-NEXT: .note.GNU-stack
 
-# SCRIPT2: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections:
-# SCRIPT2-NEXT: Name
-# SCRIPT2-NEXT: .text.bar
-# SCRIPT2-NEXT: .text.foo
+# NO-RODATA:      COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections:
+# NO-RODATA-NEXT: Name
+# NO-RODATA-NEXT: .text.bar
+# NO-RODATA-NEXT: .rela.text.bar
+# NO-RODATA-NEXT: .text.foo
+# NO-RODATA-NEXT: .rela.text.foo
 
+#--- a.s
 .weak abc
 abc:
 
@@ -52,6 +87,6 @@ abc:
 .byte 42
 
 .section .text.bar,"axG",@progbits,abc,comdat
-.quad 42
+.quad abc
 .section .text.foo,"axG",@progbits,abc,comdat
-.long 42
+.quad abc
diff --git a/lldb/bindings/interface/SBCommandInterpreterRunOptionsDocstrings.i b/lldb/bindings/interface/SBCommandInterpreterRunOptionsDocstrings.i
index b37da0535d18af31ae72c041e40871520dabc237..a4398d95ed0d1ca788bda9f76b67d25aa1c887cf 100644
--- a/lldb/bindings/interface/SBCommandInterpreterRunOptionsDocstrings.i
+++ b/lldb/bindings/interface/SBCommandInterpreterRunOptionsDocstrings.i
@@ -10,5 +10,8 @@ A default SBCommandInterpreterRunOptions object has:
 * PrintResults:   true
 * PrintErrors:    true
 * AddToHistory:   true
+* AllowRepeats    false
 
+Interactive debug sessions always allow repeats, the AllowRepeats
+run option only affects non-interactive sessions.
 ") lldb::SBCommandInterpreterRunOptions;
diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index 641b2e64d53b17284e80ac70dc4d48e487e4d7d7..1c0d717ce455cf1502052793e8abba280e07e2cf 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -284,7 +284,9 @@ class CrashLog(symbolication.Symbolicator):
         """Class that represents a binary images in a darwin crash log"""
 
         dsymForUUIDBinary = "/usr/local/bin/dsymForUUID"
-        if not os.path.exists(dsymForUUIDBinary):
+        if "LLDB_APPLE_DSYMFORUUID_EXECUTABLE" in os.environ:
+            dsymForUUIDBinary = os.environ["LLDB_APPLE_DSYMFORUUID_EXECUTABLE"]
+        elif not os.path.exists(dsymForUUIDBinary):
             try:
                 dsymForUUIDBinary = (
                     subprocess.check_output("which dsymForUUID", shell=True)
@@ -545,9 +547,9 @@ class CrashLog(symbolication.Symbolicator):
             for image in self.images:
                 image.resolve = True
         elif options.crashed_only:
+            images_to_load = []
             for thread in self.threads:
-                if thread.did_crash():
-                    images_to_load = []
+                if thread.did_crash() or thread.app_specific_backtrace:
                     for ident in thread.idents:
                         for image in self.find_images_with_identifier(ident):
                             image.resolve = True
@@ -555,11 +557,15 @@ class CrashLog(symbolication.Symbolicator):
 
         futures = []
         with tempfile.TemporaryDirectory() as obj_dir:
-            with concurrent.futures.ThreadPoolExecutor() as executor:
 
-                def add_module(image, target, obj_dir):
-                    return image, image.add_module(target, obj_dir)
+            def add_module(image, target, obj_dir):
+                return image, image.add_module(target, obj_dir)
 
+            max_worker = None
+            if options.no_parallel_image_loading:
+                max_worker = 1
+
+            with concurrent.futures.ThreadPoolExecutor(max_worker) as executor:
                 for image in images_to_load:
                     if image not in loaded_images:
                         if image.uuid == uuid.UUID(int=0):
@@ -858,7 +864,7 @@ class JSONCrashLogParser(CrashLogParser):
         thread = self.crashlog.Thread(
             len(self.crashlog.threads), True, self.crashlog.process_arch
         )
-        thread.queue = "Application Specific Backtrace"
+        thread.name = "Application Specific Backtrace"
         if self.parse_asi_backtrace(thread, json_app_specific_bts[0]):
             self.crashlog.threads.append(thread)
         else:
@@ -868,7 +874,7 @@ class JSONCrashLogParser(CrashLogParser):
         thread = self.crashlog.Thread(
             len(self.crashlog.threads), True, self.crashlog.process_arch
         )
-        thread.queue = "Last Exception Backtrace"
+        thread.name = "Last Exception Backtrace"
         self.parse_frames(thread, json_last_exc_bts)
         self.crashlog.threads.append(thread)
 
@@ -1168,11 +1174,13 @@ class TextCrashLogParser(CrashLogParser):
                 self.thread = self.crashlog.Thread(
                     idx, True, self.crashlog.process_arch
                 )
+                self.thread.name = "Application Specific Backtrace"
         elif line.startswith("Last Exception Backtrace:"):  # iOS
             self.parse_mode = self.CrashLogParseMode.THREAD
             self.app_specific_backtrace = True
             idx = 1
             self.thread = self.crashlog.Thread(idx, True, self.crashlog.process_arch)
+            self.thread.name = "Last Exception Backtrace"
         self.crashlog.info_lines.append(line.strip())
 
     def parse_thread(self, line):
@@ -1528,6 +1536,7 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
                 "file_path": crashlog_path,
                 "load_all_images": options.load_all_images,
                 "crashed_only": options.crashed_only,
+                "no_parallel_image_loading": options.no_parallel_image_loading,
             }
         )
     )
@@ -1720,6 +1729,13 @@ def CreateSymbolicateCrashLogOptions(
         help="show source for all threads, not just the crashed thread",
         default=False,
     )
+    arg_parser.add_argument(
+        "--no-parallel-image-loading",
+        dest="no_parallel_image_loading",
+        action="store_true",
+        help=argparse.SUPPRESS,
+        default=False,
+    )
     if add_interactive_options:
         arg_parser.add_argument(
             "-i",
@@ -1798,6 +1814,9 @@ def SymbolicateCrashLogs(debugger, command_args, result, is_command):
             )
         )
 
+    if "NO_PARALLEL_IMG_LOADING" in os.environ:
+        options.no_parallel_image_loading = True
+
     if options.version:
         print(debugger.GetVersionString())
         return
diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py
index 26c5c37b7371de9595316b0d317a748935eb1067..be0ed49d35904a2d0929c6a0afe7b13b09be9e96 100644
--- a/lldb/examples/python/crashlog_scripted_process.py
+++ b/lldb/examples/python/crashlog_scripted_process.py
@@ -53,6 +53,7 @@ class CrashLogScriptedProcess(ScriptedProcess):
     class CrashLogOptions:
         load_all_images = False
         crashed_only = True
+        no_parallel_image_loading = False
 
     def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData):
         super().__init__(exe_ctx, args)
@@ -84,6 +85,13 @@ class CrashLogScriptedProcess(ScriptedProcess):
             if crashed_only.GetType() == lldb.eStructuredDataTypeBoolean:
                 self.options.crashed_only = crashed_only.GetBooleanValue()
 
+        no_parallel_image_loading = args.GetValueForKey("no_parallel_image_loading")
+        if no_parallel_image_loading and no_parallel_image_loading.IsValid():
+            if no_parallel_image_loading.GetType() == lldb.eStructuredDataTypeBoolean:
+                self.options.no_parallel_image_loading = (
+                    no_parallel_image_loading.GetBooleanValue()
+                )
+
         self.pid = super().get_process_id()
         self.crashed_thread_idx = 0
         self.exception = None
@@ -165,10 +173,7 @@ class CrashLogScriptedThread(ScriptedThread):
         self.backing_thread = crashlog_thread
         self.idx = self.backing_thread.index
         self.tid = self.backing_thread.id
-        if self.backing_thread.app_specific_backtrace:
-            self.name = "Application Specific Backtrace"
-        else:
-            self.name = self.backing_thread.name
+        self.name = self.backing_thread.name
         self.queue = self.backing_thread.queue
         self.has_crashed = self.originating_process.crashed_thread_idx == self.idx
         self.create_stackframes()
diff --git a/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h b/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
index 69b969267e755abcc3c3572d90067d2076ec4a1c..0f248c926d454550a283e56108d91aa8f758847c 100644
--- a/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
+++ b/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
@@ -72,6 +72,14 @@ public:
 
   void SetSpawnThread(bool);
 
+  bool GetAllowRepeats() const;
+
+  /// By default, RunCommandInterpreter will discard repeats if the
+  /// IOHandler being used is not interactive.  Setting AllowRepeats to true
+  /// will override this behavior and always process empty lines in the input
+  /// as a repeat command.
+  void SetAllowRepeats(bool);
+
 private:
   lldb_private::CommandInterpreterRunOptions *get() const;
 
diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index e7e35e2b2bffc07f98039b8f2bcb5ab961698f2b..1e8c7c9c00536df23d5acf717eb409c7f2fb7472 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -959,9 +959,12 @@ protected:
   /// Should only be called by ValueObject::GetChildAtIndex().
   ///
   /// \return A ValueObject managed by this ValueObject's manager.
-  virtual ValueObject *CreateChildAtIndex(size_t idx,
-                                          bool synthetic_array_member,
-                                          int32_t synthetic_index);
+  virtual ValueObject *CreateChildAtIndex(size_t idx);
+
+  /// Should only be called by ValueObject::GetSyntheticArrayMember().
+  ///
+  /// \return A ValueObject managed by this ValueObject's manager.
+  virtual ValueObject *CreateSyntheticArrayMember(size_t idx);
 
   /// Should only be called by ValueObject::GetNumChildren().
   virtual llvm::Expected<uint32_t>
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 37dc0867f26c9e27820b6437f72da393a2ec2bc4..d3b3362bd0e9ecfce3a073098a85677cd09b8d01 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -79,9 +79,6 @@ public:
 
   lldb::ValueObjectSP Dereference(Status &error) override;
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index) override;
-
   lldb::ValueObjectSP GetSyntheticChildAtOffset(
       uint32_t offset, const CompilerType &type, bool can_create,
       ConstString name_const_str = ConstString()) override;
@@ -151,6 +148,13 @@ private:
   ValueObjectConstResult(ExecutionContextScope *exe_scope,
                          ValueObjectManager &manager, const Status &error);
 
+  ValueObject *CreateChildAtIndex(size_t idx) override {
+    return m_impl.CreateChildAtIndex(idx);
+  }
+  ValueObject *CreateSyntheticArrayMember(size_t idx) override {
+    return m_impl.CreateSyntheticArrayMember(idx);
+  }
+
   ValueObjectConstResult(const ValueObjectConstResult &) = delete;
   const ValueObjectConstResult &
   operator=(const ValueObjectConstResult &) = delete;
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultCast.h b/lldb/include/lldb/Core/ValueObjectConstResultCast.h
index efcbe0dc6a0bd9f645994b44fa7ebf84490a2bcd..911a08363b39353edf015025f168df4cc78b7e0f 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultCast.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResultCast.h
@@ -35,9 +35,6 @@ public:
 
   lldb::ValueObjectSP Dereference(Status &error) override;
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index) override;
-
   virtual CompilerType GetCompilerType() {
     return ValueObjectCast::GetCompilerType();
   }
@@ -61,6 +58,13 @@ private:
   friend class ValueObjectConstResult;
   friend class ValueObjectConstResultImpl;
 
+  ValueObject *CreateChildAtIndex(size_t idx) override {
+    return m_impl.CreateChildAtIndex(idx);
+  }
+  ValueObject *CreateSyntheticArrayMember(size_t idx) override {
+    return m_impl.CreateSyntheticArrayMember(idx);
+  }
+
   ValueObjectConstResultCast(const ValueObjectConstResultCast &) = delete;
   const ValueObjectConstResultCast &
   operator=(const ValueObjectConstResultCast &) = delete;
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultChild.h b/lldb/include/lldb/Core/ValueObjectConstResultChild.h
index 7e9da14e8e97f769245cdec72a76466cfe2503e4..71a3c53befe786e29d524d93abfe47cd6ae0d9f1 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultChild.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResultChild.h
@@ -41,9 +41,6 @@ public:
 
   lldb::ValueObjectSP Dereference(Status &error) override;
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index) override;
-
   virtual CompilerType GetCompilerType() {
     return ValueObjectChild::GetCompilerType();
   }
@@ -70,6 +67,13 @@ private:
   friend class ValueObjectConstResult;
   friend class ValueObjectConstResultImpl;
 
+  ValueObject *CreateChildAtIndex(size_t idx) override {
+    return m_impl.CreateChildAtIndex(idx);
+  }
+  ValueObject *CreateSyntheticArrayMember(size_t idx) override {
+    return m_impl.CreateSyntheticArrayMember(idx);
+  }
+
   ValueObjectConstResultChild(const ValueObjectConstResultChild &) = delete;
   const ValueObjectConstResultChild &
   operator=(const ValueObjectConstResultChild &) = delete;
diff --git a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h b/lldb/include/lldb/Core/ValueObjectConstResultImpl.h
index 5a7a079d3095c93da1638b5696708d61df25639b..68ba8ae7fba206cad59da7e169a428f37644fc37 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResultImpl.h
@@ -38,8 +38,8 @@ public:
 
   lldb::ValueObjectSP Dereference(Status &error);
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index);
+  ValueObject *CreateChildAtIndex(size_t idx);
+  ValueObject *CreateSyntheticArrayMember(size_t idx);
 
   lldb::ValueObjectSP
   GetSyntheticChildAtOffset(uint32_t offset, const CompilerType &type,
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index fec8566ba33d903a372a0e0fa28af6c9f614e613..d948c663a4f8bf534f94f8f8d245c6d0386fb5f2 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -49,9 +49,6 @@ public:
 
   llvm::Expected<uint32_t> CalculateNumChildren(uint32_t max) override;
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index) override;
-
   lldb::ValueObjectSP GetChildMemberWithName(llvm::StringRef name,
                                              bool can_create = true) override;
 
@@ -73,6 +70,11 @@ private:
                          ValueObjectManager &manager,
                          lldb::RegisterContextSP &reg_ctx_sp, uint32_t set_idx);
 
+  ValueObject *CreateChildAtIndex(size_t idx) override;
+  ValueObject *CreateSyntheticArrayMember(size_t idx) override {
+    return nullptr;
+  }
+
   // For ValueObject only
   ValueObjectRegisterSet(const ValueObjectRegisterSet &) = delete;
   const ValueObjectRegisterSet &
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index 4662f395a4dde97bf523b5e4f240ba0a612f3096..7087dcc1d1bec542ab4c2ea088ed79aaf962bbc0 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -66,9 +66,6 @@ public:
 
   llvm::Expected<uint32_t> CalculateNumChildren(uint32_t max) override;
 
-  ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
-                                  int32_t synthetic_index) override;
-
   lldb::ValueType GetValueType() const override;
 
   ConstString GetTypeName() override;
@@ -95,6 +92,11 @@ protected:
 private:
   ValueObjectVTable(ValueObject &parent);
 
+  ValueObject *CreateChildAtIndex(size_t idx) override;
+  ValueObject *CreateSyntheticArrayMember(size_t idx) override {
+    return nullptr;
+  }
+
   // For ValueObject only
   ValueObjectVTable(const ValueObjectVTable &) = delete;
   const ValueObjectVTable &operator=(const ValueObjectVTable &) = delete;
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index 8863523b2e31f327a2045b88ae6d2a8d3b9a61cc..48f6618ab0e3929147da0cb2d9606445f12467b2 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -93,15 +93,20 @@ public:
   /// \param[in] add_to_history
   ///    If \b true add the commands to the command history. If \b false, don't
   ///    add them.
+  /// \param[in] handle_repeats
+  ///    If \b true then treat empty lines as repeat commands even if the
+  ///    interpreter is non-interactive.
   CommandInterpreterRunOptions(LazyBool stop_on_continue,
                                LazyBool stop_on_error, LazyBool stop_on_crash,
                                LazyBool echo_commands, LazyBool echo_comments,
                                LazyBool print_results, LazyBool print_errors,
-                               LazyBool add_to_history)
+                               LazyBool add_to_history,
+                               LazyBool handle_repeats)
       : m_stop_on_continue(stop_on_continue), m_stop_on_error(stop_on_error),
         m_stop_on_crash(stop_on_crash), m_echo_commands(echo_commands),
         m_echo_comment_commands(echo_comments), m_print_results(print_results),
-        m_print_errors(print_errors), m_add_to_history(add_to_history) {}
+        m_print_errors(print_errors), m_add_to_history(add_to_history),
+        m_allow_repeats(handle_repeats) {}
 
   CommandInterpreterRunOptions() = default;
 
@@ -183,6 +188,12 @@ public:
     m_spawn_thread = spawn_thread ? eLazyBoolYes : eLazyBoolNo;
   }
 
+  bool GetAllowRepeats() const { return DefaultToNo(m_allow_repeats); }
+
+  void SetAllowRepeats(bool allow_repeats) {
+    m_allow_repeats = allow_repeats ? eLazyBoolYes : eLazyBoolNo;
+  }
+
   LazyBool m_stop_on_continue = eLazyBoolCalculate;
   LazyBool m_stop_on_error = eLazyBoolCalculate;
   LazyBool m_stop_on_crash = eLazyBoolCalculate;
@@ -193,6 +204,7 @@ public:
   LazyBool m_add_to_history = eLazyBoolCalculate;
   LazyBool m_auto_handle_events;
   LazyBool m_spawn_thread;
+  LazyBool m_allow_repeats = eLazyBoolCalculate;
 
 private:
   static bool DefaultToYes(LazyBool flag) {
diff --git a/lldb/source/API/SBBreakpoint.cpp b/lldb/source/API/SBBreakpoint.cpp
index f1fb6f904f5f0c7bc9f1b2252a7e737e45227e94..3d908047f9455b5941327eb784e7cc491bcad0cd 100644
--- a/lldb/source/API/SBBreakpoint.cpp
+++ b/lldb/source/API/SBBreakpoint.cpp
@@ -714,7 +714,7 @@ void SBBreakpoint::GetNames(SBStringList &names) {
         bkpt_sp->GetTarget().GetAPIMutex());
     std::vector<std::string> names_vec;
     bkpt_sp->GetNames(names_vec);
-    for (std::string name : names_vec) {
+    for (const std::string &name : names_vec) {
       names.AppendString(name.c_str());
     }
   }
diff --git a/lldb/source/API/SBCommandInterpreterRunOptions.cpp b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
index 6c6b2aa15a79235ba15cd18bcbbdb874f7b747e3..0c7581d6f1f5bd79a1bc23b60af5948f53d4e130 100644
--- a/lldb/source/API/SBCommandInterpreterRunOptions.cpp
+++ b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
@@ -164,6 +164,18 @@ void SBCommandInterpreterRunOptions::SetSpawnThread(bool spawn_thread) {
   m_opaque_up->SetSpawnThread(spawn_thread);
 }
 
+bool SBCommandInterpreterRunOptions::GetAllowRepeats() const {
+  LLDB_INSTRUMENT_VA(this);
+
+  return m_opaque_up->GetAllowRepeats();
+}
+
+void SBCommandInterpreterRunOptions::SetAllowRepeats(bool allow_repeats) {
+  LLDB_INSTRUMENT_VA(this, allow_repeats);
+
+  m_opaque_up->SetAllowRepeats(allow_repeats);
+}
+
 lldb_private::CommandInterpreterRunOptions *
 SBCommandInterpreterRunOptions::get() const {
   return m_opaque_up.get();
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index 962ce9ba83cc7764c9ecf264aa4d919bf97e3aa1..adb9e645610ba45af5d4b776770a8a9f31d6b946 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -1147,7 +1147,7 @@ void SBTarget::GetBreakpointNames(SBStringList &names) {
 
     std::vector<std::string> name_vec;
     target_sp->GetBreakpointNames(name_vec);
-    for (auto name : name_vec)
+    for (const auto &name : name_vec)
       names.AppendString(name.c_str());
   }
 }
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index ae845e92762b970cacea7b2251c5b1807c6291aa..dc80d435ad44490514032d00d89f638138800d97 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -885,7 +885,7 @@ void Breakpoint::GetDescription(Stream *s, lldb::DescriptionLevel level,
         s->Printf("Names:");
         s->EOL();
         s->IndentMore();
-        for (std::string name : m_name_list) {
+        for (const std::string &name : m_name_list) {
           s->Indent();
           s->Printf("%s\n", name.c_str());
         }
diff --git a/lldb/source/Breakpoint/BreakpointIDList.cpp b/lldb/source/Breakpoint/BreakpointIDList.cpp
index 97af1d40eb7a5824b0dad6c9d4a8ebb610237ce8..5fc9f95a75db1214af439a1d067c81f336ece620 100644
--- a/lldb/source/Breakpoint/BreakpointIDList.cpp
+++ b/lldb/source/Breakpoint/BreakpointIDList.cpp
@@ -259,7 +259,7 @@ llvm::Error BreakpointIDList::FindAndReplaceIDRanges(
     
     if (!names_found.empty()) {
       for (BreakpointSP bkpt_sp : target->GetBreakpointList().Breakpoints()) {
-        for (std::string name : names_found) {
+        for (const std::string &name : names_found) {
           if (bkpt_sp->MatchesName(name.c_str())) {
             StreamString canonical_id_str;
             BreakpointID::GetCanonicalReference(
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index c5c434a941b3411064affe97b8611d9a071933aa..f32cb82faa006f98704d84fc07dcbfce9b6007e4 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -382,7 +382,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
       // subclass do it and cache the result for quick future access.
-      m_children.SetChildAtIndex(idx, CreateChildAtIndex(idx, false, 0));
+      m_children.SetChildAtIndex(idx, CreateChildAtIndex(idx));
     }
 
     ValueObject *child = m_children.GetChildAtIndex(idx);
@@ -488,14 +488,10 @@ void ValueObject::SetNumChildren(uint32_t num_children) {
   m_children.SetChildrenCount(num_children);
 }
 
-ValueObject *ValueObject::CreateChildAtIndex(size_t idx,
-                                             bool synthetic_array_member,
-                                             int32_t synthetic_index) {
-  ValueObject *valobj = nullptr;
-
+ValueObject *ValueObject::CreateChildAtIndex(size_t idx) {
   bool omit_empty_base_classes = true;
-  bool ignore_array_bounds = synthetic_array_member;
-  std::string child_name_str;
+  bool ignore_array_bounds = false;
+  std::string child_name;
   uint32_t child_byte_size = 0;
   int32_t child_byte_offset = 0;
   uint32_t child_bitfield_bit_size = 0;
@@ -503,51 +499,74 @@ ValueObject *ValueObject::CreateChildAtIndex(size_t idx,
   bool child_is_base_class = false;
   bool child_is_deref_of_parent = false;
   uint64_t language_flags = 0;
-
-  const bool transparent_pointers = !synthetic_array_member;
+  const bool transparent_pointers = true;
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
 
   auto child_compiler_type_or_err =
       GetCompilerType().GetChildCompilerTypeAtIndex(
           &exe_ctx, idx, transparent_pointers, omit_empty_base_classes,
-          ignore_array_bounds, child_name_str, child_byte_size,
-          child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset,
+          ignore_array_bounds, child_name, child_byte_size, child_byte_offset,
+          child_bitfield_bit_size, child_bitfield_bit_offset,
           child_is_base_class, child_is_deref_of_parent, this, language_flags);
-  CompilerType child_compiler_type;
-  if (!child_compiler_type_or_err)
+  if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) {
     LLDB_LOG_ERROR(GetLog(LLDBLog::Types),
                    child_compiler_type_or_err.takeError(),
                    "could not find child: {0}");
-  else
-    child_compiler_type = *child_compiler_type_or_err;
+    return nullptr;
+  }
+
+  return new ValueObjectChild(
+      *this, *child_compiler_type_or_err, ConstString(child_name),
+      child_byte_size, child_byte_offset, child_bitfield_bit_size,
+      child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent,
+      eAddressTypeInvalid, language_flags);
+}
+
+ValueObject *ValueObject::CreateSyntheticArrayMember(size_t idx) {
+  bool omit_empty_base_classes = true;
+  bool ignore_array_bounds = true;
+  std::string child_name;
+  uint32_t child_byte_size = 0;
+  int32_t child_byte_offset = 0;
+  uint32_t child_bitfield_bit_size = 0;
+  uint32_t child_bitfield_bit_offset = 0;
+  bool child_is_base_class = false;
+  bool child_is_deref_of_parent = false;
+  uint64_t language_flags = 0;
+  const bool transparent_pointers = false;
 
-  if (child_compiler_type) {
-    if (synthetic_index)
-      child_byte_offset += child_byte_size * synthetic_index;
+  ExecutionContext exe_ctx(GetExecutionContextRef());
 
-    ConstString child_name;
-    if (!child_name_str.empty())
-      child_name.SetCString(child_name_str.c_str());
+  auto child_compiler_type_or_err =
+      GetCompilerType().GetChildCompilerTypeAtIndex(
+          &exe_ctx, 0, transparent_pointers, omit_empty_base_classes,
+          ignore_array_bounds, child_name, child_byte_size, child_byte_offset,
+          child_bitfield_bit_size, child_bitfield_bit_offset,
+          child_is_base_class, child_is_deref_of_parent, this, language_flags);
+  if (!child_compiler_type_or_err) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Types),
+                   child_compiler_type_or_err.takeError(),
+                   "could not find child: {0}");
+    return nullptr;
+  }
+
+  if (child_compiler_type_or_err->IsValid()) {
+    child_byte_offset += child_byte_size * idx;
 
-    valobj = new ValueObjectChild(
-        *this, child_compiler_type, child_name, child_byte_size,
-        child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset,
-        child_is_base_class, child_is_deref_of_parent, eAddressTypeInvalid,
-        language_flags);
+    return new ValueObjectChild(
+        *this, *child_compiler_type_or_err, ConstString(child_name),
+        child_byte_size, child_byte_offset, child_bitfield_bit_size,
+        child_bitfield_bit_offset, child_is_base_class,
+        child_is_deref_of_parent, eAddressTypeInvalid, language_flags);
   }
 
   // In case of an incomplete type, try to use the ValueObject's
   // synthetic value to create the child ValueObject.
-  if (!valobj && synthetic_array_member) {
-    if (ValueObjectSP synth_valobj_sp = GetSyntheticValue()) {
-      valobj = synth_valobj_sp
-                   ->GetChildAtIndex(synthetic_index, synthetic_array_member)
-                   .get();
-    }
-  }
+  if (ValueObjectSP synth_valobj_sp = GetSyntheticValue())
+    return synth_valobj_sp->GetChildAtIndex(idx, /*can_create=*/true).get();
 
-  return valobj;
+  return nullptr;
 }
 
 bool ValueObject::GetSummaryAsCString(TypeSummaryImpl *summary_ptr,
@@ -1616,7 +1635,7 @@ ValueObjectSP ValueObject::GetSyntheticArrayMember(size_t index,
       ValueObject *synthetic_child;
       // We haven't made a synthetic array member for INDEX yet, so lets make
       // one and cache it for any future reference.
-      synthetic_child = CreateChildAtIndex(0, true, index);
+      synthetic_child = CreateSyntheticArrayMember(index);
 
       // Cache the value if we got one back...
       if (synthetic_child) {
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 8ac2c1cac2f665499764d24ff0baf09b96cfc361..879d3c3f6b0372536c129905b7ef0fa2ce1a9628 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -267,12 +267,6 @@ lldb::addr_t ValueObjectConstResult::GetAddressOf(bool scalar_is_load_address,
   return m_impl.GetAddressOf(scalar_is_load_address, address_type);
 }
 
-ValueObject *ValueObjectConstResult::CreateChildAtIndex(
-    size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
-  return m_impl.CreateChildAtIndex(idx, synthetic_array_member,
-                                   synthetic_index);
-}
-
 size_t ValueObjectConstResult::GetPointeeData(DataExtractor &data,
                                               uint32_t item_idx,
                                               uint32_t item_count) {
diff --git a/lldb/source/Core/ValueObjectConstResultCast.cpp b/lldb/source/Core/ValueObjectConstResultCast.cpp
index fceb2635f876f0a539a4f24e980ab309263625ee..bf7a12dc682366a165905ffb40f5e32b04a88438 100644
--- a/lldb/source/Core/ValueObjectConstResultCast.cpp
+++ b/lldb/source/Core/ValueObjectConstResultCast.cpp
@@ -44,12 +44,6 @@ lldb::ValueObjectSP ValueObjectConstResultCast::AddressOf(Status &error) {
   return m_impl.AddressOf(error);
 }
 
-ValueObject *ValueObjectConstResultCast::CreateChildAtIndex(
-    size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
-  return m_impl.CreateChildAtIndex(idx, synthetic_array_member,
-                                   synthetic_index);
-}
-
 size_t ValueObjectConstResultCast::GetPointeeData(DataExtractor &data,
                                                   uint32_t item_idx,
                                                   uint32_t item_count) {
diff --git a/lldb/source/Core/ValueObjectConstResultChild.cpp b/lldb/source/Core/ValueObjectConstResultChild.cpp
index 36bf11a0b73af27d8c45b3d8d26e77c83711b40c..39fc0c9fbb35b889e2b78abf0256463b3c9ebdd8 100644
--- a/lldb/source/Core/ValueObjectConstResultChild.cpp
+++ b/lldb/source/Core/ValueObjectConstResultChild.cpp
@@ -56,12 +56,6 @@ lldb::addr_t ValueObjectConstResultChild::GetAddressOf(
   return m_impl.GetAddressOf(scalar_is_load_address, address_type);
 }
 
-ValueObject *ValueObjectConstResultChild::CreateChildAtIndex(
-    size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
-  return m_impl.CreateChildAtIndex(idx, synthetic_array_member,
-                                   synthetic_index);
-}
-
 size_t ValueObjectConstResultChild::GetPointeeData(DataExtractor &data,
                                                    uint32_t item_idx,
                                                    uint32_t item_count) {
diff --git a/lldb/source/Core/ValueObjectConstResultImpl.cpp b/lldb/source/Core/ValueObjectConstResultImpl.cpp
index 493980d7ea96043f06f17fc92ada7c1f5a011aac..2a7c9077007653f9be4c93417f2b270fbd999abd 100644
--- a/lldb/source/Core/ValueObjectConstResultImpl.cpp
+++ b/lldb/source/Core/ValueObjectConstResultImpl.cpp
@@ -46,18 +46,15 @@ lldb::ValueObjectSP ValueObjectConstResultImpl::Dereference(Status &error) {
   return m_impl_backend->ValueObject::Dereference(error);
 }
 
-ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex(
-    size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
+ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex(size_t idx) {
   if (m_impl_backend == nullptr)
     return nullptr;
 
   m_impl_backend->UpdateValueIfNeeded(false);
 
-  ValueObjectConstResultChild *valobj = nullptr;
-
   bool omit_empty_base_classes = true;
-  bool ignore_array_bounds = synthetic_array_member;
-  std::string child_name_str;
+  bool ignore_array_bounds = false;
+  std::string child_name;
   uint32_t child_byte_size = 0;
   int32_t child_byte_offset = 0;
   uint32_t child_bitfield_bit_size = 0;
@@ -65,56 +62,100 @@ ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex(
   bool child_is_base_class = false;
   bool child_is_deref_of_parent = false;
   uint64_t language_flags;
-
-  const bool transparent_pointers = !synthetic_array_member;
+  const bool transparent_pointers = true;
   CompilerType compiler_type = m_impl_backend->GetCompilerType();
 
   ExecutionContext exe_ctx(m_impl_backend->GetExecutionContextRef());
 
   auto child_compiler_type_or_err = compiler_type.GetChildCompilerTypeAtIndex(
       &exe_ctx, idx, transparent_pointers, omit_empty_base_classes,
-      ignore_array_bounds, child_name_str, child_byte_size, child_byte_offset,
+      ignore_array_bounds, child_name, child_byte_size, child_byte_offset,
       child_bitfield_bit_size, child_bitfield_bit_offset, child_is_base_class,
       child_is_deref_of_parent, m_impl_backend, language_flags);
-  CompilerType child_compiler_type;
-  if (!child_compiler_type_or_err)
+
+  // One might think we should check that the size of the children
+  // is always strictly positive, hence we could avoid creating a
+  // ValueObject if that's not the case, but it turns out there
+  // are languages out there which allow zero-size types with
+  // children (e.g. Swift).
+  if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) {
     LLDB_LOG_ERROR(GetLog(LLDBLog::Types),
                    child_compiler_type_or_err.takeError(),
                    "could not find child: {0}");
-  else
-    child_compiler_type = *child_compiler_type_or_err;
+    return nullptr;
+  }
 
+  lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS;
+  // Transfer the live address (with offset) to the child.  But if
+  // the parent is a pointer, the live address is where that pointer
+  // value lives in memory, so the children live addresses aren't
+  // offsets from that value, they are just other load addresses that
+  // are recorded in the Value of the child ValueObjects.
+  if (m_live_address != LLDB_INVALID_ADDRESS && !compiler_type.IsPointerType())
+    child_live_addr = m_live_address + child_byte_offset;
+
+  return new ValueObjectConstResultChild(
+      *m_impl_backend, *child_compiler_type_or_err, ConstString(child_name),
+      child_byte_size, child_byte_offset, child_bitfield_bit_size,
+      child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent,
+      child_live_addr, language_flags);
+}
+
+ValueObject *
+ValueObjectConstResultImpl::CreateSyntheticArrayMember(size_t idx) {
+  if (m_impl_backend == nullptr)
+    return nullptr;
+
+  m_impl_backend->UpdateValueIfNeeded(false);
+
+  bool omit_empty_base_classes = true;
+  bool ignore_array_bounds = true;
+  std::string child_name;
+  uint32_t child_byte_size = 0;
+  int32_t child_byte_offset = 0;
+  uint32_t child_bitfield_bit_size = 0;
+  uint32_t child_bitfield_bit_offset = 0;
+  bool child_is_base_class = false;
+  bool child_is_deref_of_parent = false;
+  uint64_t language_flags;
+
+  const bool transparent_pointers = false;
+  CompilerType compiler_type = m_impl_backend->GetCompilerType();
+
+  ExecutionContext exe_ctx(m_impl_backend->GetExecutionContextRef());
+
+  auto child_compiler_type_or_err = compiler_type.GetChildCompilerTypeAtIndex(
+      &exe_ctx, 0, transparent_pointers, omit_empty_base_classes,
+      ignore_array_bounds, child_name, child_byte_size, child_byte_offset,
+      child_bitfield_bit_size, child_bitfield_bit_offset, child_is_base_class,
+      child_is_deref_of_parent, m_impl_backend, language_flags);
   // One might think we should check that the size of the children
   // is always strictly positive, hence we could avoid creating a
   // ValueObject if that's not the case, but it turns out there
   // are languages out there which allow zero-size types with
   // children (e.g. Swift).
-  if (child_compiler_type) {
-    if (synthetic_index)
-      child_byte_offset += child_byte_size * synthetic_index;
-
-    ConstString child_name;
-    if (!child_name_str.empty())
-      child_name.SetCString(child_name_str.c_str());
-
-    lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS;
-    // Transfer the live address (with offset) to the child.  But if
-    // the parent is a pointer, the live address is where that pointer
-    // value lives in memory, so the children live addresses aren't
-    // offsets from that value, they are just other load addresses that
-    // are recorded in the Value of the child ValueObjects.
-    if (m_live_address != LLDB_INVALID_ADDRESS) {
-      if (!compiler_type.IsPointerType())
-        child_live_addr = m_live_address + child_byte_offset;
-    }
-    valobj = new ValueObjectConstResultChild(
-        *m_impl_backend, child_compiler_type, child_name, child_byte_size,
-        child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset,
-        child_is_base_class, child_is_deref_of_parent, child_live_addr,
-        language_flags);
+  if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Types),
+                   child_compiler_type_or_err.takeError(),
+                   "could not find child: {0}");
+    return nullptr;
   }
 
-  return valobj;
+  child_byte_offset += child_byte_size * idx;
+
+  lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS;
+  // Transfer the live address (with offset) to the child.  But if
+  // the parent is a pointer, the live address is where that pointer
+  // value lives in memory, so the children live addresses aren't
+  // offsets from that value, they are just other load addresses that
+  // are recorded in the Value of the child ValueObjects.
+  if (m_live_address != LLDB_INVALID_ADDRESS && !compiler_type.IsPointerType())
+    child_live_addr = m_live_address + child_byte_offset;
+  return new ValueObjectConstResultChild(
+      *m_impl_backend, *child_compiler_type_or_err, ConstString(child_name),
+      child_byte_size, child_byte_offset, child_bitfield_bit_size,
+      child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent,
+      child_live_addr, language_flags);
 }
 
 lldb::ValueObjectSP ValueObjectConstResultImpl::GetSyntheticChildAtOffset(
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index 9d9da8bb119df2fb22d1bb857c57632c3f18ced7..8b5557a0178baf8c7043209cc501a40c017f7ee6 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -115,17 +115,13 @@ bool ValueObjectRegisterSet::UpdateValue() {
   return m_error.Success();
 }
 
-ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
-    size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
-  ValueObject *valobj = nullptr;
+ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(size_t idx) {
   if (m_reg_ctx_sp && m_reg_set) {
-    uint32_t num_children = GetNumChildrenIgnoringErrors();
-    if (idx < num_children)
-      valobj = new ValueObjectRegister(
-          *this, m_reg_ctx_sp,
-          m_reg_ctx_sp->GetRegisterInfoAtIndex(m_reg_set->registers[idx]));
+    return new ValueObjectRegister(
+        *this, m_reg_ctx_sp,
+        m_reg_ctx_sp->GetRegisterInfoAtIndex(m_reg_set->registers[idx]));
   }
-  return valobj;
+  return nullptr;
 }
 
 lldb::ValueObjectSP
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 9bfe8dc9700d306c9e3ced8c03521dca0bad7c40..72e198d08c9ca83759f42e1ef8436135a7d2e868 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -185,11 +185,7 @@ ConstString ValueObjectVTable::GetDisplayTypeName() {
 
 bool ValueObjectVTable::IsInScope() { return GetParent()->IsInScope(); }
 
-ValueObject *ValueObjectVTable::CreateChildAtIndex(size_t idx,
-                                                   bool synthetic_array_member,
-                                                   int32_t synthetic_index) {
-  if (synthetic_array_member)
-    return nullptr;
+ValueObject *ValueObjectVTable::CreateChildAtIndex(size_t idx) {
   return new ValueObjectVTableChild(*this, idx, m_addr_size);
 }
 
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 80d03c84fadbdd63b1198f451360bbaed535deda..444e44b3928919fd4b833ef6af7c4e9b8730c923 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -94,51 +94,38 @@ void DWARFExpression::SetRegisterKind(RegisterKind reg_kind) {
   m_reg_kind = reg_kind;
 }
 
-
-static bool ReadRegisterValueAsScalar(RegisterContext *reg_ctx,
-                                      lldb::RegisterKind reg_kind,
-                                      uint32_t reg_num, Status *error_ptr,
-                                      Value &value) {
-  if (reg_ctx == nullptr) {
-    if (error_ptr)
-      error_ptr->SetErrorString("No register context in frame.\n");
-  } else {
-    uint32_t native_reg =
-        reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num);
-    if (native_reg == LLDB_INVALID_REGNUM) {
-      if (error_ptr)
-        error_ptr->SetErrorStringWithFormat("Unable to convert register "
-                                            "kind=%u reg_num=%u to a native "
-                                            "register number.\n",
-                                            reg_kind, reg_num);
-    } else {
-      const RegisterInfo *reg_info =
-          reg_ctx->GetRegisterInfoAtIndex(native_reg);
-      RegisterValue reg_value;
-      if (reg_ctx->ReadRegister(reg_info, reg_value)) {
-        if (reg_value.GetScalarValue(value.GetScalar())) {
-          value.SetValueType(Value::ValueType::Scalar);
-          value.SetContext(Value::ContextType::RegisterInfo,
-                           const_cast<RegisterInfo *>(reg_info));
-          if (error_ptr)
-            error_ptr->Clear();
-          return true;
-        } else {
-          // If we get this error, then we need to implement a value buffer in
-          // the dwarf expression evaluation function...
-          if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
-                "register %s can't be converted to a scalar value",
-                reg_info->name);
-        }
-      } else {
-        if (error_ptr)
-          error_ptr->SetErrorStringWithFormat("register %s is not available",
-                                              reg_info->name);
-      }
+static llvm::Error ReadRegisterValueAsScalar(RegisterContext *reg_ctx,
+                                             lldb::RegisterKind reg_kind,
+                                             uint32_t reg_num, Value &value) {
+  if (reg_ctx == nullptr)
+    return llvm::createStringError("no register context in frame");
+
+  const uint32_t native_reg =
+      reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num);
+  if (native_reg == LLDB_INVALID_REGNUM)
+    return llvm::createStringError(
+        "unable to convert register kind=%u reg_num=%u to a native "
+        "register number",
+        reg_kind, reg_num);
+
+  const RegisterInfo *reg_info = reg_ctx->GetRegisterInfoAtIndex(native_reg);
+  RegisterValue reg_value;
+  if (reg_ctx->ReadRegister(reg_info, reg_value)) {
+    if (reg_value.GetScalarValue(value.GetScalar())) {
+      value.SetValueType(Value::ValueType::Scalar);
+      value.SetContext(Value::ContextType::RegisterInfo,
+                       const_cast<RegisterInfo *>(reg_info));
+      return llvm::Error::success();
     }
+
+    // If we get this error, then we need to implement a value buffer in
+    // the dwarf expression evaluation function...
+    return llvm::createStringError(
+        "register %s can't be converted to a scalar value", reg_info->name);
   }
-  return false;
+
+  return llvm::createStringError("register %s is not available",
+                                 reg_info->name);
 }
 
 /// Return the length in bytes of the set of operands for \p op. No guarantees
@@ -782,7 +769,6 @@ void UpdateValueTypeFromLocationDescription(Log *log, const DWARFUnit *dwarf_cu,
 ///
 /// \param exe_ctx Pointer to the execution context
 /// \param module_sp shared_ptr contains the module if we have one
-/// \param error_ptr pointer to Status object if we have one
 /// \param dw_op_type C-style string used to vary the error output
 /// \param file_addr the file address we are trying to resolve and turn into a
 ///                  load address
@@ -793,32 +779,22 @@ void UpdateValueTypeFromLocationDescription(Log *log, const DWARFUnit *dwarf_cu,
 ///          the load address succeed or an empty Optinal otherwise. If
 ///          check_sectionoffset is true we consider LLDB_INVALID_ADDRESS a
 ///          success if so_addr.IsSectionOffset() is true.
-static std::optional<lldb::addr_t>
+static llvm::Expected<lldb::addr_t>
 ResolveLoadAddress(ExecutionContext *exe_ctx, lldb::ModuleSP &module_sp,
-                   Status *error_ptr, const char *dw_op_type,
-                   lldb::addr_t file_addr, Address &so_addr,
-                   bool check_sectionoffset = false) {
-  if (!module_sp) {
-    if (error_ptr)
-      error_ptr->SetErrorStringWithFormat(
-          "need module to resolve file address for %s", dw_op_type);
-    return {};
-  }
+                   const char *dw_op_type, lldb::addr_t file_addr,
+                   Address &so_addr, bool check_sectionoffset = false) {
+  if (!module_sp)
+    return llvm::createStringError("need module to resolve file address for %s",
+                                   dw_op_type);
 
-  if (!module_sp->ResolveFileAddress(file_addr, so_addr)) {
-    if (error_ptr)
-      error_ptr->SetErrorString("failed to resolve file address in module");
-    return {};
-  }
+  if (!module_sp->ResolveFileAddress(file_addr, so_addr))
+    return llvm::createStringError("failed to resolve file address in module");
 
-  addr_t load_addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr());
+  const addr_t load_addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr());
 
   if (load_addr == LLDB_INVALID_ADDRESS &&
-      (check_sectionoffset && !so_addr.IsSectionOffset())) {
-    if (error_ptr)
-      error_ptr->SetErrorString("failed to resolve load address");
-    return {};
-  }
+      (check_sectionoffset && !so_addr.IsSectionOffset()))
+    return llvm::createStringError("failed to resolve load address");
 
   return load_addr;
 }
@@ -912,6 +888,14 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
                 DW_OP_value_to_name(op));
     }
 
+    if (std::optional<unsigned> arity =
+            llvm::dwarf::OperationArity(static_cast<LocationAtom>(op))) {
+      if (stack.size() < *arity)
+        return llvm::createStringError(
+            "%s needs at least %d stack entries (stack has %d entries)",
+            DW_OP_value_to_name(op), *arity, stack.size());
+    }
+
     switch (op) {
     // The DW_OP_addr operation has a single operand that encodes a machine
     // address and whose size is the size of an address on the target machine.
@@ -973,7 +957,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     case DW_OP_deref: {
       if (stack.empty())
         return llvm::createStringError(
-            "Expression stack empty for DW_OP_deref.");
+            "expression stack empty for DW_OP_deref");
       Value::ValueType value_type = stack.back().GetValueType();
       switch (value_type) {
       case Value::ValueType::HostAddress: {
@@ -988,12 +972,11 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
             LLDB_INVALID_ADDRESS);
 
         Address so_addr;
-        Status load_err;
         auto maybe_load_addr = ResolveLoadAddress(
-            exe_ctx, module_sp, &load_err, "DW_OP_deref", file_addr, so_addr);
+            exe_ctx, module_sp, "DW_OP_deref", file_addr, so_addr);
 
         if (!maybe_load_addr)
-          return load_err.ToError();
+          return maybe_load_addr.takeError();
 
         stack.back().GetScalar() = *maybe_load_addr;
         // Fall through to load address promotion code below.
@@ -1023,16 +1006,16 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
                   pointer_addr, error.AsCString());
             }
           } else {
-            return llvm::createStringError("NULL process for DW_OP_deref.\n");
+            return llvm::createStringError("NULL process for DW_OP_deref");
           }
         } else {
           return llvm::createStringError(
-              "NULL execution context for DW_OP_deref.\n");
+              "NULL execution context for DW_OP_deref");
         }
         break;
 
       case Value::ValueType::Invalid:
-        return llvm::createStringError("Invalid value type for DW_OP_deref.\n");
+        return llvm::createStringError("invalid value type for DW_OP_deref");
       }
 
     } break;
@@ -1052,7 +1035,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     case DW_OP_deref_size: {
       if (stack.empty()) {
         return llvm::createStringError(
-            "Expression stack empty for DW_OP_deref_size.");
+            "expression stack empty for DW_OP_deref_size");
       }
       uint8_t size = opcodes.GetU8(&offset);
       if (size > 8) {
@@ -1105,14 +1088,12 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
         auto file_addr =
             stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
         Address so_addr;
-        Status resolve_err;
-        auto maybe_load_addr =
-            ResolveLoadAddress(exe_ctx, module_sp, &resolve_err,
-                               "DW_OP_deref_size", file_addr, so_addr,
-                               /*check_sectionoffset=*/true);
+        auto maybe_load_addr = ResolveLoadAddress(
+            exe_ctx, module_sp, "DW_OP_deref_size", file_addr, so_addr,
+            /*check_sectionoffset=*/true);
 
         if (!maybe_load_addr)
-          return resolve_err.ToError();
+          return maybe_load_addr.takeError();
 
         addr_t load_addr = *maybe_load_addr;
 
@@ -1164,18 +1145,17 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
             }
           } else {
 
-            return llvm::createStringError(
-                "NULL process for DW_OP_deref_size.\n");
+            return llvm::createStringError("NULL process for DW_OP_deref_size");
           }
         } else {
           return llvm::createStringError(
-              "NULL execution context for DW_OP_deref_size.\n");
+              "NULL execution context for DW_OP_deref_size");
         }
         break;
 
       case Value::ValueType::Invalid:
 
-        return llvm::createStringError("Invalid value for DW_OP_deref_size.\n");
+        return llvm::createStringError("invalid value for DW_OP_deref_size");
       }
 
     } break;
@@ -1196,9 +1176,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // extended to the size of an address on the target machine before being
     // pushed on the expression stack.
     case DW_OP_xderef_size:
-
-      return llvm::createStringError(
-          "Unimplemented opcode: DW_OP_xderef_size.");
+      return llvm::createStringError("unimplemented opcode: DW_OP_xderef_size");
     // OPCODE: DW_OP_xderef
     // OPERANDS: none
     // DESCRIPTION: Provides an extended dereference mechanism. The entry at
@@ -1210,7 +1188,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // retrieved from the dereferenced address is the size of an address on the
     // target machine.
     case DW_OP_xderef:
-      return llvm::createStringError("Unimplemented opcode: DW_OP_xderef.");
+      return llvm::createStringError("unimplemented opcode: DW_OP_xderef");
 
     // All DW_OP_constXXX opcodes have a single operand as noted below:
     //
@@ -1263,7 +1241,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: duplicates the value at the top of the stack
     case DW_OP_dup:
       if (stack.empty()) {
-        return llvm::createStringError("Expression stack empty for DW_OP_dup.");
+        return llvm::createStringError("expression stack empty for DW_OP_dup");
       } else
         stack.push_back(stack.back());
       break;
@@ -1273,8 +1251,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the value at the top of the stack
     case DW_OP_drop:
       if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack empty for DW_OP_drop.");
+        return llvm::createStringError("expression stack empty for DW_OP_drop");
       } else
         stack.pop_back();
       break;
@@ -1284,11 +1261,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: Duplicates the entry currently second in the stack at
     // the top of the stack.
     case DW_OP_over:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_over.");
-      } else
-        stack.push_back(stack[stack.size() - 2]);
+      stack.push_back(stack[stack.size() - 2]);
       break;
 
     // OPCODE: DW_OP_pick
@@ -1311,14 +1284,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the stack becomes the second stack entry, and the second entry
     // becomes the top of the stack
     case DW_OP_swap:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_swap.");
-      } else {
-        tmp = stack.back();
-        stack.back() = stack[stack.size() - 2];
-        stack[stack.size() - 2] = tmp;
-      }
+      tmp = stack.back();
+      stack.back() = stack[stack.size() - 2];
+      stack[stack.size() - 2] = tmp;
       break;
 
     // OPCODE: DW_OP_rot
@@ -1327,18 +1295,13 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // the top of the stack becomes the third stack entry, the second entry
     // becomes the top of the stack, and the third entry becomes the second
     // entry.
-    case DW_OP_rot:
-      if (stack.size() < 3) {
-        return llvm::createStringError(
-            "Expression stack needs at least 3 items for DW_OP_rot.");
-      } else {
-        size_t last_idx = stack.size() - 1;
-        Value old_top = stack[last_idx];
-        stack[last_idx] = stack[last_idx - 1];
-        stack[last_idx - 1] = stack[last_idx - 2];
-        stack[last_idx - 2] = old_top;
-      }
-      break;
+    case DW_OP_rot: {
+      size_t last_idx = stack.size() - 1;
+      Value old_top = stack[last_idx];
+      stack[last_idx] = stack[last_idx - 1];
+      stack[last_idx - 1] = stack[last_idx - 2];
+      stack[last_idx - 2] = old_top;
+    } break;
 
     // OPCODE: DW_OP_abs
     // OPERANDS: none
@@ -1346,12 +1309,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // value and pushes its absolute value. If the absolute value can not be
     // represented, the result is undefined.
     case DW_OP_abs:
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_abs.");
-      } else if (!stack.back().ResolveValue(exe_ctx).AbsoluteValue()) {
+      if (!stack.back().ResolveValue(exe_ctx).AbsoluteValue()) {
         return llvm::createStringError(
-            "Failed to take the absolute value of the first stack item.");
+            "failed to take the absolute value of the first stack item");
       }
       break;
 
@@ -1360,15 +1320,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack values, performs a bitwise and
     // operation on the two, and pushes the result.
     case DW_OP_and:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_and.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) & tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) & tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_div
@@ -1376,43 +1331,32 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack values, divides the former second
     // entry by the former top of the stack using signed division, and pushes
     // the result.
-    case DW_OP_div:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_div.");
-      } else {
-        tmp = stack.back();
-        if (tmp.ResolveValue(exe_ctx).IsZero()) {
-          return llvm::createStringError("Divide by zero.");
-        } else {
-          stack.pop_back();
-          Scalar divisor, dividend;
-          divisor = tmp.ResolveValue(exe_ctx);
-          dividend = stack.back().ResolveValue(exe_ctx);
-          divisor.MakeSigned();
-          dividend.MakeSigned();
-          stack.back() = dividend / divisor;
-          if (!stack.back().ResolveValue(exe_ctx).IsValid()) {
-            return llvm::createStringError("Divide failed.");
-          }
-        }
-      }
-      break;
+    case DW_OP_div: {
+      tmp = stack.back();
+      if (tmp.ResolveValue(exe_ctx).IsZero())
+        return llvm::createStringError("divide by zero");
+
+      stack.pop_back();
+      Scalar divisor, dividend;
+      divisor = tmp.ResolveValue(exe_ctx);
+      dividend = stack.back().ResolveValue(exe_ctx);
+      divisor.MakeSigned();
+      dividend.MakeSigned();
+      stack.back() = dividend / divisor;
+
+      if (!stack.back().ResolveValue(exe_ctx).IsValid())
+        return llvm::createStringError("divide failed");
+    } break;
 
     // OPCODE: DW_OP_minus
     // OPERANDS: none
     // DESCRIPTION: pops the top two stack values, subtracts the former top
     // of the stack from the former second entry, and pushes the result.
     case DW_OP_minus:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_minus.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) - tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) - tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_mod
@@ -1421,15 +1365,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // the calculation: former second stack entry modulo the former top of the
     // stack.
     case DW_OP_mod:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_mod.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) % tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) % tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_mul
@@ -1437,28 +1376,18 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack entries, multiplies them
     // together, and pushes the result.
     case DW_OP_mul:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_mul.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) * tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) * tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_neg
     // OPERANDS: none
     // DESCRIPTION: pops the top stack entry, and pushes its negation.
     case DW_OP_neg:
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_neg.");
-      } else {
-        if (!stack.back().ResolveValue(exe_ctx).UnaryNegate())
-          return llvm::createStringError("Unary negate failed.");
-      }
+      if (!stack.back().ResolveValue(exe_ctx).UnaryNegate())
+        return llvm::createStringError("unary negate failed");
       break;
 
     // OPCODE: DW_OP_not
@@ -1466,14 +1395,8 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top stack entry, and pushes its bitwise
     // complement
     case DW_OP_not:
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_not.");
-      } else {
-        if (!stack.back().ResolveValue(exe_ctx).OnesComplement()) {
-          return llvm::createStringError("Logical NOT failed.");
-        }
-      }
+      if (!stack.back().ResolveValue(exe_ctx).OnesComplement())
+        return llvm::createStringError("logical NOT failed");
       break;
 
     // OPCODE: DW_OP_or
@@ -1481,15 +1404,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack entries, performs a bitwise or
     // operation on the two, and pushes the result.
     case DW_OP_or:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_or.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) | tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) | tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_plus
@@ -1497,34 +1415,22 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack entries, adds them together, and
     // pushes the result.
     case DW_OP_plus:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_plus.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().GetScalar() += tmp.GetScalar();
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().GetScalar() += tmp.GetScalar();
       break;
 
     // OPCODE: DW_OP_plus_uconst
     // OPERANDS: none
     // DESCRIPTION: pops the top stack entry, adds it to the unsigned LEB128
     // constant operand and pushes the result.
-    case DW_OP_plus_uconst:
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_plus_uconst.");
-      } else {
-        const uint64_t uconst_value = opcodes.GetULEB128(&offset);
-        // Implicit conversion from a UINT to a Scalar...
-        stack.back().GetScalar() += uconst_value;
-        if (!stack.back().GetScalar().IsValid()) {
-
-          return llvm::createStringError("DW_OP_plus_uconst failed.");
-        }
-      }
-      break;
+    case DW_OP_plus_uconst: {
+      const uint64_t uconst_value = opcodes.GetULEB128(&offset);
+      // Implicit conversion from a UINT to a Scalar...
+      stack.back().GetScalar() += uconst_value;
+      if (!stack.back().GetScalar().IsValid())
+        return llvm::createStringError("DW_OP_plus_uconst failed");
+    } break;
 
     // OPCODE: DW_OP_shl
     // OPERANDS: none
@@ -1532,14 +1438,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // second entry left by the number of bits specified by the former top of
     // the stack, and pushes the result.
     case DW_OP_shl:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_shl.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) <<= tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) <<= tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_shr
@@ -1548,17 +1449,11 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // entry right logically (filling with zero bits) by the number of bits
     // specified by the former top of the stack, and pushes the result.
     case DW_OP_shr:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_shr.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        if (!stack.back().ResolveValue(exe_ctx).ShiftRightLogical(
-                tmp.ResolveValue(exe_ctx))) {
-          return llvm::createStringError("DW_OP_shr failed.");
-        }
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      if (!stack.back().ResolveValue(exe_ctx).ShiftRightLogical(
+              tmp.ResolveValue(exe_ctx)))
+        return llvm::createStringError("DW_OP_shr failed");
       break;
 
     // OPCODE: DW_OP_shra
@@ -1568,14 +1463,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // sign for the result) by the number of bits specified by the former top
     // of the stack, and pushes the result.
     case DW_OP_shra:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_shra.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) >>= tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) >>= tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_xor
@@ -1583,15 +1473,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: pops the top two stack entries, performs the bitwise
     // exclusive-or operation on the two, and pushes the result.
     case DW_OP_xor:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_xor.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) ^ tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) ^ tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_skip
@@ -1622,30 +1507,25 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // value popped is not the constant 0, the 2-byte constant operand is the
     // number of bytes of the DWARF expression to skip forward or backward from
     // the current operation, beginning after the 2-byte constant.
-    case DW_OP_bra:
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_bra.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        int16_t bra_offset = (int16_t)opcodes.GetU16(&offset);
-        Scalar zero(0);
-        if (tmp.ResolveValue(exe_ctx) != zero) {
-          lldb::offset_t new_offset = offset + bra_offset;
-          // New offset can point at the end of the data, in this case we should
-          // terminate the DWARF expression evaluation (will happen in the loop
-          // condition).
-          if (new_offset <= opcodes.GetByteSize())
-            offset = new_offset;
-          else {
-            return llvm::createStringError(llvm::formatv(
-                "Invalid opcode offset in DW_OP_bra: {0}+({1}) > {2}", offset,
-                bra_offset, opcodes.GetByteSize()));
-          }
+    case DW_OP_bra: {
+      tmp = stack.back();
+      stack.pop_back();
+      int16_t bra_offset = (int16_t)opcodes.GetU16(&offset);
+      Scalar zero(0);
+      if (tmp.ResolveValue(exe_ctx) != zero) {
+        lldb::offset_t new_offset = offset + bra_offset;
+        // New offset can point at the end of the data, in this case we should
+        // terminate the DWARF expression evaluation (will happen in the loop
+        // condition).
+        if (new_offset <= opcodes.GetByteSize())
+          offset = new_offset;
+        else {
+          return llvm::createStringError(llvm::formatv(
+              "Invalid opcode offset in DW_OP_bra: {0}+({1}) > {2}", offset,
+              bra_offset, opcodes.GetByteSize()));
         }
       }
-      break;
+    } break;
 
     // OPCODE: DW_OP_eq
     // OPERANDS: none
@@ -1655,15 +1535,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_eq:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_eq.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) == tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) == tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_ge
@@ -1674,15 +1549,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_ge:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_ge.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) >= tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) >= tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_gt
@@ -1693,15 +1563,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_gt:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_gt.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) > tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) > tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_le
@@ -1712,15 +1577,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_le:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_le.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) <= tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) <= tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_lt
@@ -1731,15 +1591,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_lt:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_lt.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) < tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) < tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_ne
@@ -1750,15 +1605,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // of the operation is true or the constant value 0 if the result of the
     // operation is false.
     case DW_OP_ne:
-      if (stack.size() < 2) {
-        return llvm::createStringError(
-            "Expression stack needs at least 2 items for DW_OP_ne.");
-      } else {
-        tmp = stack.back();
-        stack.pop_back();
-        stack.back().ResolveValue(exe_ctx) =
-            stack.back().ResolveValue(exe_ctx) != tmp.ResolveValue(exe_ctx);
-      }
+      tmp = stack.back();
+      stack.pop_back();
+      stack.back().ResolveValue(exe_ctx) =
+          stack.back().ResolveValue(exe_ctx) != tmp.ResolveValue(exe_ctx);
       break;
 
     // OPCODE: DW_OP_litn
@@ -1839,11 +1689,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
       dwarf4_location_description_kind = Register;
       reg_num = op - DW_OP_reg0;
 
-      Status read_err;
-      if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, tmp))
-        stack.push_back(tmp);
-      else
-        return read_err.ToError();
+      if (llvm::Error err =
+              ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp))
+        return err;
+      stack.push_back(tmp);
     } break;
     // OPCODE: DW_OP_regx
     // OPERANDS:
@@ -1853,10 +1702,10 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
       dwarf4_location_description_kind = Register;
       reg_num = opcodes.GetULEB128(&offset);
       Status read_err;
-      if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, tmp))
-        stack.push_back(tmp);
-      else
-        return read_err.ToError();
+      if (llvm::Error err =
+              ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp))
+        return err;
+      stack.push_back(tmp);
     } break;
 
     // OPCODE: DW_OP_bregN
@@ -1897,17 +1746,15 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     case DW_OP_breg30:
     case DW_OP_breg31: {
       reg_num = op - DW_OP_breg0;
-
-      Status read_err;
-      if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err,
-                                    tmp)) {
-        int64_t breg_offset = opcodes.GetSLEB128(&offset);
-        tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset;
-        tmp.ClearContext();
-        stack.push_back(tmp);
-        stack.back().SetValueType(Value::ValueType::LoadAddress);
-      } else
-        return read_err.ToError();
+      if (llvm::Error err =
+              ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp))
+        return err;
+
+      int64_t breg_offset = opcodes.GetSLEB128(&offset);
+      tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset;
+      tmp.ClearContext();
+      stack.push_back(tmp);
+      stack.back().SetValueType(Value::ValueType::LoadAddress);
     } break;
     // OPCODE: DW_OP_bregx
     // OPERANDS: 2
@@ -1917,17 +1764,15 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // N plus an offset.
     case DW_OP_bregx: {
       reg_num = opcodes.GetULEB128(&offset);
-
-      Status read_err;
-      if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err,
-                                    tmp)) {
-        int64_t breg_offset = opcodes.GetSLEB128(&offset);
-        tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset;
-        tmp.ClearContext();
-        stack.push_back(tmp);
-        stack.back().SetValueType(Value::ValueType::LoadAddress);
-      } else
-        return read_err.ToError();
+      if (llvm::Error err =
+              ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp))
+        return err;
+
+      int64_t breg_offset = opcodes.GetSLEB128(&offset);
+      tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset;
+      tmp.ClearContext();
+      stack.push_back(tmp);
+      stack.back().SetValueType(Value::ValueType::LoadAddress);
     } break;
 
     case DW_OP_fbreg:
@@ -1944,11 +1789,11 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
             return fb_err.ToError();
         } else {
           return llvm::createStringError(
-              "Invalid stack frame in context for DW_OP_fbreg opcode.");
+              "invalid stack frame in context for DW_OP_fbreg opcode");
         }
       } else {
         return llvm::createStringError(
-            "NULL execution context for DW_OP_fbreg.\n");
+            "NULL execution context for DW_OP_fbreg");
       }
 
       break;
@@ -2098,7 +1943,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
         // Reset for the next piece.
         dwarf4_location_description_kind = Memory;
         return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_bit_piece.");
+            "expression stack needs at least 1 item for DW_OP_bit_piece");
       } else {
         UpdateValueTypeFromLocationDescription(
             log, dwarf_cu, dwarf4_location_description_kind, &stack.back());
@@ -2109,7 +1954,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
         switch (stack.back().GetValueType()) {
         case Value::ValueType::Invalid:
           return llvm::createStringError(
-              "unable to extract bit value from invalid value.");
+              "unable to extract bit value from invalid value");
         case Value::ValueType::Scalar: {
           if (!stack.back().GetScalar().ExtractBitfield(piece_bit_size,
                                                         piece_bit_offset)) {
@@ -2147,7 +1992,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
 
       if (!data) {
         LLDB_LOG(log, "Evaluate_DW_OP_implicit_value: could not be read data");
-        return llvm::createStringError("Could not evaluate %s.",
+        return llvm::createStringError("could not evaluate %s",
                                        DW_OP_value_to_name(op));
       }
 
@@ -2198,7 +2043,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // the stack by the called expression may be used as return values by prior
     // agreement between the calling and called expressions.
     case DW_OP_call2:
-      return llvm::createStringError("Unimplemented opcode DW_OP_call2.");
+      return llvm::createStringError("unimplemented opcode DW_OP_call2");
     // OPCODE: DW_OP_call4
     // OPERANDS: 1
     //      uint32_t compile unit relative offset of a DIE
@@ -2219,8 +2064,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // the stack by the called expression may be used as return values by prior
     // agreement between the calling and called expressions.
     case DW_OP_call4:
-
-      return llvm::createStringError("Unimplemented opcode DW_OP_call4.");
+      return llvm::createStringError("unimplemented opcode DW_OP_call4");
 
     // OPCODE: DW_OP_stack_value
     // OPERANDS: None
@@ -2229,10 +2073,6 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // value to be used.  This is the actual object value and not the location.
     case DW_OP_stack_value:
       dwarf4_location_description_kind = Implicit;
-      if (stack.empty()) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_stack_value.");
-      }
       stack.back().SetValueType(Value::ValueType::Scalar);
       break;
 
@@ -2244,10 +2084,6 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // DESCRIPTION: Pop the top stack element, convert it to a
     // different type, and push the result.
     case DW_OP_convert: {
-      if (stack.size() < 1) {
-        return llvm::createStringError(
-            "Expression stack needs at least 1 item for DW_OP_convert.");
-      }
       const uint64_t die_offset = opcodes.GetULEB128(&offset);
       uint64_t bit_size;
       bool sign;
@@ -2256,7 +2092,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
         // machine and an unspecified signedness. Scalar has no
         // "unspecified signedness", so we use unsigned types.
         if (!module_sp)
-          return llvm::createStringError("No module");
+          return llvm::createStringError("no module");
         sign = false;
         bit_size = module_sp->GetArchitecture().GetAddressByteSize() * 8;
         if (!bit_size)
@@ -2269,7 +2105,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
         DWARFDIE die = const_cast<DWARFUnit *>(dwarf_cu)->GetDIE(abs_die_offset);
         if (!die)
           return llvm::createStringError(
-              "Cannot resolve DW_OP_convert type DIE");
+              "cannot resolve DW_OP_convert type DIE");
         uint64_t encoding =
             die.GetAttributeValueAsUnsigned(DW_AT_encoding, DW_ATE_hi_user);
         bit_size = die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8;
@@ -2277,7 +2113,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
           bit_size = die.GetAttributeValueAsUnsigned(DW_AT_bit_size, 0);
         if (!bit_size)
           return llvm::createStringError(
-              "Unsupported type size in DW_OP_convert");
+              "unsupported type size in DW_OP_convert");
         switch (encoding) {
         case DW_ATE_signed:
         case DW_ATE_signed_char:
@@ -2289,7 +2125,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
           break;
         default:
           return llvm::createStringError(
-              "Unsupported encoding in DW_OP_convert");
+              "unsupported encoding in DW_OP_convert");
         }
       }
       Scalar &top = stack.back().ResolveValue(exe_ctx);
@@ -2313,13 +2149,13 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
           stack.back().SetValueType(Value::ValueType::LoadAddress);
         } else {
           return llvm::createStringError(
-              "Stack frame does not include a canonical "
+              "stack frame does not include a canonical "
               "frame address for DW_OP_call_frame_cfa "
-              "opcode.");
+              "opcode");
         }
       } else {
-        return llvm::createStringError("Invalid stack frame in context for "
-                                       "DW_OP_call_frame_cfa opcode.");
+        return llvm::createStringError("unvalid stack frame in context for "
+                                       "DW_OP_call_frame_cfa opcode");
       }
       break;
 
@@ -2334,18 +2170,18 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
       if (stack.size() < 1) {
         if (op == DW_OP_form_tls_address)
           return llvm::createStringError(
-              "DW_OP_form_tls_address needs an argument.");
+              "DW_OP_form_tls_address needs an argument");
         else
           return llvm::createStringError(
-              "DW_OP_GNU_push_tls_address needs an argument.");
+              "DW_OP_GNU_push_tls_address needs an argument");
       }
 
       if (!exe_ctx || !module_sp)
-        return llvm::createStringError("No context to evaluate TLS within.");
+        return llvm::createStringError("no context to evaluate TLS within");
 
       Thread *thread = exe_ctx->GetThreadPtr();
       if (!thread)
-        return llvm::createStringError("No thread to evaluate TLS within.");
+        return llvm::createStringError("no thread to evaluate TLS within");
 
       // Lookup the TLS block address for this thread and module.
       const addr_t tls_file_addr =
@@ -2355,7 +2191,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
 
       if (tls_load_addr == LLDB_INVALID_ADDRESS)
         return llvm::createStringError(
-            "No TLS data currently exists for this thread.");
+            "no TLS data currently exists for this thread");
 
       stack.back().GetScalar() = tls_load_addr;
       stack.back().SetValueType(Value::ValueType::LoadAddress);
@@ -2430,7 +2266,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     if (pieces.GetBuffer().GetByteSize())
       return pieces;
 
-    return llvm::createStringError("Stack empty after evaluation.");
+    return llvm::createStringError("stack empty after evaluation");
   }
 
   UpdateValueTypeFromLocationDescription(
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index acd6294cb3f423a98b32f015628f51de676dbffa..da995de1407c4edc4101e3899624ed25b1969e21 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -2707,7 +2707,8 @@ enum {
   eHandleCommandFlagEchoCommentCommand = (1u << 3),
   eHandleCommandFlagPrintResult = (1u << 4),
   eHandleCommandFlagPrintErrors = (1u << 5),
-  eHandleCommandFlagStopOnCrash = (1u << 6)
+  eHandleCommandFlagStopOnCrash = (1u << 6),
+  eHandleCommandFlagAllowRepeats = (1u << 7)
 };
 
 void CommandInterpreter::HandleCommandsFromFile(
@@ -3129,14 +3130,19 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
       return;
 
   const bool is_interactive = io_handler.GetIsInteractive();
-  if (!is_interactive) {
+  const bool allow_repeats =
+      io_handler.GetFlags().Test(eHandleCommandFlagAllowRepeats);
+
+  if (!is_interactive && !allow_repeats) {
     // When we are not interactive, don't execute blank lines. This will happen
     // sourcing a commands file. We don't want blank lines to repeat the
     // previous command and cause any errors to occur (like redefining an
     // alias, get an error and stop parsing the commands file).
+    // But obey the AllowRepeats flag if the user has set it.
     if (line.empty())
       return;
-
+  }
+  if (!is_interactive) {
     // When using a non-interactive file handle (like when sourcing commands
     // from a file) we need to echo the command out so we don't just see the
     // command output and no command...
@@ -3388,6 +3394,8 @@ CommandInterpreter::GetIOHandler(bool force_create,
         flags |= eHandleCommandFlagPrintResult;
       if (options->m_print_errors != eLazyBoolNo)
         flags |= eHandleCommandFlagPrintErrors;
+      if (options->m_allow_repeats == eLazyBoolYes)
+        flags |= eHandleCommandFlagAllowRepeats;
     } else {
       flags = eHandleCommandFlagEchoCommand | eHandleCommandFlagPrintResult |
               eHandleCommandFlagPrintErrors;
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm.cpp
index 5ad2f7a8e9455b191338eb37f9e1779ffc5831ee..4668c25eab08365e43fa2c42811572f708db07a3 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm.cpp
@@ -506,7 +506,7 @@ uint32_t NativeRegisterContextLinux_arm::SetHardwareWatchpoint(
       return LLDB_INVALID_INDEX32;
     else if (watch_mask <= 0x02)
       size = 2;
-    else if (watch_mask <= 0x04)
+    else
       size = 4;
 
     addr = addr & (~0x03);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 8a47eed3d7cbe1fbae1af7daba8772d077f74ced..187370eb36cae46fab6e2e05b8b43ac3f1dfb2c5 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -539,9 +539,8 @@ bool GDBRemoteCommunication::DecompressPacket() {
       else if (m_compression_type == CompressionType::ZlibDeflate)
         scratchbuf_size = compression_decode_scratch_buffer_size (COMPRESSION_ZLIB);
       else if (m_compression_type == CompressionType::LZMA)
-        scratchbuf_size = compression_decode_scratch_buffer_size (COMPRESSION_LZMA);
-      else if (m_compression_type == CompressionType::LZFSE)
-        scratchbuf_size = compression_decode_scratch_buffer_size (COMPRESSION_LZFSE);
+        scratchbuf_size =
+            compression_decode_scratch_buffer_size(COMPRESSION_LZMA);
       if (scratchbuf_size > 0) {
         m_decompression_scratch = (void*) malloc (scratchbuf_size);
         m_decompression_scratch_type = m_compression_type;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
index e144cf0f9bd94e10bbef65ff3fa79428aeab265d..66db396279e0630ebfb0ce0c9e46cae48d804b0d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
@@ -60,8 +60,6 @@ public:
 
   virtual ConstString GetDIEClassTemplateParams(const DWARFDIE &die) = 0;
 
-  virtual lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) = 0;
-
   static std::optional<SymbolFile::ArrayInfo>
   ParseChildArrayInfo(const DWARFDIE &parent_die,
                       const ExecutionContext *exe_ctx = nullptr);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 7d7e835c3d732b235794ea37e478437b6dab6d74..579a538af36345106fdf92742c58f9fb04067c74 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -154,26 +154,6 @@ static bool TagIsRecordType(dw_tag_t tag) {
   }
 }
 
-static bool IsForwardDeclaration(const DWARFDIE &die,
-                                 const ParsedDWARFTypeAttributes &attrs,
-                                 LanguageType cu_language) {
-  if (attrs.is_forward_declaration)
-    return true;
-
-  // Work around an issue with clang at the moment where forward
-  // declarations for objective C classes are emitted as:
-  //  DW_TAG_structure_type [2]
-  //  DW_AT_name( "ForwardObjcClass" )
-  //  DW_AT_byte_size( 0x00 )
-  //  DW_AT_decl_file( "..." )
-  //  DW_AT_decl_line( 1 )
-  //
-  // Note that there is no DW_AT_declaration and there are no children,
-  // and the byte size is zero.
-  return attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
-         !die.HasChildren() && cu_language == eLanguageTypeObjC;
-}
-
 TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
                                                      const DWARFDIE &die,
                                                      Log *log) {
@@ -269,9 +249,11 @@ static void ForcefullyCompleteType(CompilerType type) {
 /// This function serves a similar purpose as RequireCompleteType above, but it
 /// avoids completing the type if it is not immediately necessary. It only
 /// ensures we _can_ complete the type later.
-void DWARFASTParserClang::PrepareContextToReceiveMembers(
-    clang::DeclContext *decl_ctx, const DWARFDIE &decl_ctx_die,
-    const DWARFDIE &die, const char *type_name_cstr) {
+static void PrepareContextToReceiveMembers(TypeSystemClang &ast,
+                                           ClangASTImporter &ast_importer,
+                                           clang::DeclContext *decl_ctx,
+                                           DWARFDIE die,
+                                           const char *type_name_cstr) {
   auto *tag_decl_ctx = clang::dyn_cast<clang::TagDecl>(decl_ctx);
   if (!tag_decl_ctx)
     return; // Non-tag context are always ready.
@@ -286,8 +268,7 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers(
   // gmodules case), we can complete the type by doing a full import.
 
   // If this type was not imported from an external AST, there's nothing to do.
-  CompilerType type = m_ast.GetTypeForDecl(tag_decl_ctx);
-  ClangASTImporter &ast_importer = GetClangASTImporter();
+  CompilerType type = ast.GetTypeForDecl(tag_decl_ctx);
   if (type && ast_importer.CanImport(type)) {
     auto qual_type = ClangUtil::GetQualType(type);
     if (ast_importer.RequireCompleteType(qual_type))
@@ -298,13 +279,6 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers(
         type_name_cstr ? type_name_cstr : "", die.GetOffset());
   }
 
-  // By searching for the definition DIE of the decl_ctx type, we will either:
-  // 1. Found the the definition DIE and start its definition with
-  // TypeSystemClang::StartTagDeclarationDefinition.
-  // 2. Unable to find it, then need to forcefully complete it.
-  FindDefinitionTypeForDIE(decl_ctx_die);
-  if (tag_decl_ctx->isCompleteDefinition() || tag_decl_ctx->isBeingDefined())
-    return;
   // We don't have a type definition and/or the import failed. We must
   // forcefully complete the type to avoid crashes.
   ForcefullyCompleteType(type);
@@ -650,11 +624,10 @@ DWARFASTParserClang::ParseTypeModifier(const SymbolContext &sc,
   if (tag == DW_TAG_typedef) {
     // DeclContext will be populated when the clang type is materialized in
     // Type::ResolveCompilerType.
-    DWARFDIE decl_ctx_die;
-    clang::DeclContext *decl_ctx =
-        GetClangDeclContextContainingDIE(die, &decl_ctx_die);
-    PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
-                                   attrs.name.GetCString());
+    PrepareContextToReceiveMembers(
+        m_ast, GetClangASTImporter(),
+        GetClangDeclContextContainingDIE(die, nullptr), die,
+        attrs.name.GetCString());
 
     if (attrs.type.IsValid()) {
       // Try to parse a typedef from the (DWARF embedded in the) Clang
@@ -1134,6 +1107,32 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
         // struct and see if this is actually a C++ method
         Type *class_type = dwarf->ResolveType(decl_ctx_die);
         if (class_type) {
+          if (class_type->GetID() != decl_ctx_die.GetID() ||
+              IsClangModuleFwdDecl(decl_ctx_die)) {
+
+            // We uniqued the parent class of this function to another
+            // class so we now need to associate all dies under
+            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
+            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
+
+            if (class_type_die) {
+              std::vector<DWARFDIE> failures;
+
+              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
+                                         class_type, failures);
+
+              // FIXME do something with these failures that's
+              // smarter than just dropping them on the ground.
+              // Unfortunately classes don't like having stuff added
+              // to them after their definitions are complete...
+
+              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
+              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
+                return type_ptr->shared_from_this();
+              }
+            }
+          }
+
           if (attrs.specification.IsValid()) {
             // We have a specification which we are going to base our
             // function prototype off of, so we need this type to be
@@ -1268,39 +1267,6 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
               }
             }
           }
-          // By here, we should have already completed the c++ class_type
-          // because if either specification or abstract_origin is present, we
-          // call GetClangDeclContextForDIE to resolve the DW_TAG_subprogram
-          // refered by this one until we reached the DW_TAG_subprogram without
-          // specification or abstract_origin (the else branch above). Then the
-          // above GetFullCompilerType() will complete the class_type if it's
-          // not completed yet. After that, we will have the mapping from DIEs
-          // in class_type_die to DeclContexts in m_die_to_decl_ctx.
-          if (class_type->GetID() != decl_ctx_die.GetID() ||
-              IsClangModuleFwdDecl(decl_ctx_die)) {
-
-            // We uniqued the parent class of this function to another
-            // class so we now need to associate all dies under
-            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
-            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
-
-            if (class_type_die) {
-              std::vector<DWARFDIE> failures;
-
-              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
-                                         class_type, failures);
-
-              // FIXME do something with these failures that's
-              // smarter than just dropping them on the ground.
-              // Unfortunately classes don't like having stuff added
-              // to them after their definitions are complete...
-
-              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
-              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
-                return type_ptr->shared_from_this();
-              }
-            }
-          }
         }
       }
     }
@@ -1673,93 +1639,6 @@ DWARFASTParserClang::GetCPlusPlusQualifiedName(const DWARFDIE &die) {
   return qualified_name;
 }
 
-lldb_private::Type *
-DWARFASTParserClang::FindDefinitionTypeForDIE(const DWARFDIE &die) {
-  SymbolFileDWARF *dwarf = die.GetDWARF();
-  ParsedDWARFTypeAttributes attrs(die);
-  bool is_forward_declaration = IsForwardDeclaration(
-      die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU()));
-  if (!is_forward_declaration)
-    return dwarf->GetDIEToType()[die.GetDIE()];
-
-  const dw_tag_t tag = die.Tag();
-  TypeSP type_sp;
-  Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
-  if (log) {
-    dwarf->GetObjectFile()->GetModule()->LogMessage(
-        log,
-        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-        "forward declaration DIE, trying to find definition DIE",
-        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-        attrs.name.GetCString());
-  }
-  // We haven't parse definition die for this type, starting to search for it.
-  // After we found the definition die, the GetDeclarationDIEToDefinitionDIE()
-  // map will have the new mapping from this declaration die to definition die.
-  if (attrs.class_language == eLanguageTypeObjC ||
-      attrs.class_language == eLanguageTypeObjC_plus_plus) {
-    if (!attrs.is_complete_objc_class &&
-        die.Supports_DW_AT_APPLE_objc_complete_type()) {
-      // We have a valid eSymbolTypeObjCClass class symbol whose name
-      // matches the current objective C class that we are trying to find
-      // and this DIE isn't the complete definition (we checked
-      // is_complete_objc_class above and know it is false), so the real
-      // definition is in here somewhere
-      type_sp =
-          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
-
-      if (!type_sp) {
-        SymbolFileDWARFDebugMap *debug_map_symfile =
-            dwarf->GetDebugMapSymfile();
-        if (debug_map_symfile) {
-          // We weren't able to find a full declaration in this DWARF,
-          // see if we have a declaration anywhere else...
-          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
-              die, attrs.name, true);
-        }
-      }
-
-      if (type_sp && log) {
-        dwarf->GetObjectFile()->GetModule()->LogMessage(
-            log,
-            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
-            "incomplete objc type, complete type is {5:x8}",
-            static_cast<void *>(this), die.GetOffset(),
-            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
-            type_sp->GetID());
-      }
-    }
-  }
-
-  type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
-  if (!type_sp) {
-    SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
-    if (debug_map_symfile) {
-      // We weren't able to find a full declaration in this DWARF, see
-      // if we have a declaration anywhere else...
-      type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
-    }
-    if (type_sp && log) {
-      dwarf->GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-          "forward declaration, complete type is {4:x8}",
-          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-          attrs.name.GetCString(), type_sp->GetID());
-    }
-  }
-
-  if (!type_sp && log) {
-    dwarf->GetObjectFile()->GetModule()->LogMessage(
-        log,
-        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-        "forward declaration, unable to find definition DIE for it",
-        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-        attrs.name.GetCString());
-  }
-  return type_sp.get();
-}
-
 TypeSP
 DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
                                            const DWARFDIE &die,
@@ -1771,10 +1650,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   LanguageType cu_language = SymbolFileDWARF::GetLanguage(*die.GetCU());
   Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
 
+  // UniqueDWARFASTType is large, so don't create a local variables on the
+  // stack, put it on the heap. This function is often called recursively and
+  // clang isn't good at sharing the stack space for variables in different
+  // blocks.
+  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
+
   ConstString unique_typename(attrs.name);
   Declaration unique_decl(attrs.decl);
-  uint64_t byte_size = attrs.byte_size.value_or(0);
-  attrs.is_forward_declaration = IsForwardDeclaration(die, attrs, cu_language);
 
   if (attrs.name) {
     if (Language::LanguageIsCPlusPlus(cu_language)) {
@@ -1787,42 +1670,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       unique_decl.Clear();
     }
 
-    if (UniqueDWARFASTType *unique_ast_entry_type =
-            dwarf->GetUniqueDWARFASTTypeMap().Find(
-                unique_typename, die, unique_decl, byte_size,
-                attrs.is_forward_declaration)) {
-      type_sp = unique_ast_entry_type->m_type_sp;
+    if (dwarf->GetUniqueDWARFASTTypeMap().Find(
+            unique_typename, die, unique_decl, attrs.byte_size.value_or(-1),
+            *unique_ast_entry_up)) {
+      type_sp = unique_ast_entry_up->m_type_sp;
       if (type_sp) {
         dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
         LinkDeclContextToDIE(
-            GetCachedClangDeclContextForDIE(unique_ast_entry_type->m_die), die);
-        if (!attrs.is_forward_declaration) {
-          // If the DIE being parsed in this function is a definition and the
-          // entry in the map is a declaration, then we need to update the entry
-          // to point to the definition DIE.
-          if (unique_ast_entry_type->m_is_forward_declaration) {
-            unique_ast_entry_type->m_die = die;
-            unique_ast_entry_type->m_byte_size = byte_size;
-            unique_ast_entry_type->m_declaration = unique_decl;
-            unique_ast_entry_type->m_is_forward_declaration = false;
-            // Need to update Type ID to refer to the definition DIE. because
-            // it's used in ParseSubroutine to determine if we need to copy cxx
-            // method types from a declaration DIE to this definition DIE.
-            type_sp->SetID(die.GetID());
-            clang_type = type_sp->GetForwardCompilerType();
-            if (attrs.class_language != eLanguageTypeObjC &&
-                attrs.class_language != eLanguageTypeObjC_plus_plus)
-              TypeSystemClang::StartTagDeclarationDefinition(clang_type);
-
-            CompilerType compiler_type_no_qualifiers =
-                ClangUtil::RemoveFastQualifiers(clang_type);
-            auto result = dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
-                compiler_type_no_qualifiers.GetOpaqueQualType(),
-                *die.GetDIERef());
-            if (!result.second)
-              result.first->second = *die.GetDIERef();
-          }
-        }
+            GetCachedClangDeclContextForDIE(unique_ast_entry_up->m_die), die);
         return type_sp;
       }
     }
@@ -1844,21 +1699,125 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
     default_accessibility = eAccessPrivate;
   }
 
+  if (attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
+      !die.HasChildren() && cu_language == eLanguageTypeObjC) {
+    // Work around an issue with clang at the moment where forward
+    // declarations for objective C classes are emitted as:
+    //  DW_TAG_structure_type [2]
+    //  DW_AT_name( "ForwardObjcClass" )
+    //  DW_AT_byte_size( 0x00 )
+    //  DW_AT_decl_file( "..." )
+    //  DW_AT_decl_line( 1 )
+    //
+    // Note that there is no DW_AT_declaration and there are no children,
+    // and the byte size is zero.
+    attrs.is_forward_declaration = true;
+  }
+
+  if (attrs.class_language == eLanguageTypeObjC ||
+      attrs.class_language == eLanguageTypeObjC_plus_plus) {
+    if (!attrs.is_complete_objc_class &&
+        die.Supports_DW_AT_APPLE_objc_complete_type()) {
+      // We have a valid eSymbolTypeObjCClass class symbol whose name
+      // matches the current objective C class that we are trying to find
+      // and this DIE isn't the complete definition (we checked
+      // is_complete_objc_class above and know it is false), so the real
+      // definition is in here somewhere
+      type_sp =
+          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
+
+      if (!type_sp) {
+        SymbolFileDWARFDebugMap *debug_map_symfile =
+            dwarf->GetDebugMapSymfile();
+        if (debug_map_symfile) {
+          // We weren't able to find a full declaration in this DWARF,
+          // see if we have a declaration anywhere else...
+          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
+              die, attrs.name, true);
+        }
+      }
+
+      if (type_sp) {
+        if (log) {
+          dwarf->GetObjectFile()->GetModule()->LogMessage(
+              log,
+              "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
+              "incomplete objc type, complete type is {5:x8}",
+              static_cast<void *>(this), die.GetOffset(),
+              DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
+              type_sp->GetID());
+        }
+
+        // We found a real definition for this type elsewhere so lets use
+        // it and cache the fact that we found a complete type for this
+        // die
+        dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
+        return type_sp;
+      }
+    }
+  }
+
   if (attrs.is_forward_declaration) {
+    // We have a forward declaration to a type and we need to try and
+    // find a full declaration. We look in the current type index just in
+    // case we have a forward declaration followed by an actual
+    // declarations in the DWARF. If this fails, we need to look
+    // elsewhere...
+    if (log) {
+      dwarf->GetObjectFile()->GetModule()->LogMessage(
+          log,
+          "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
+          "forward declaration, trying to find complete type",
+          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
+          tag, attrs.name.GetCString());
+    }
+
     // See if the type comes from a Clang module and if so, track down
     // that type.
     type_sp = ParseTypeFromClangModule(sc, die, log);
     if (type_sp)
       return type_sp;
-  }
 
+    // type_sp = FindDefinitionTypeForDIE (dwarf_cu, die,
+    // type_name_const_str);
+    type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
+
+    if (!type_sp) {
+      SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
+      if (debug_map_symfile) {
+        // We weren't able to find a full declaration in this DWARF, see
+        // if we have a declaration anywhere else...
+        type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
+      }
+    }
+
+    if (type_sp) {
+      if (log) {
+        dwarf->GetObjectFile()->GetModule()->LogMessage(
+            log,
+            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
+            "forward declaration, complete type is {5:x8}",
+            static_cast<void *>(this), die.GetOffset(),
+            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
+            type_sp->GetID());
+      }
+
+      // We found a real definition for this type elsewhere so lets use
+      // it and cache the fact that we found a complete type for this die
+      dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
+      clang::DeclContext *defn_decl_ctx =
+          GetCachedClangDeclContextForDIE(dwarf->GetDIE(type_sp->GetID()));
+      if (defn_decl_ctx)
+        LinkDeclContextToDIE(defn_decl_ctx, die);
+      return type_sp;
+    }
+  }
   assert(tag_decl_kind != -1);
   UNUSED_IF_ASSERT_DISABLED(tag_decl_kind);
-  DWARFDIE decl_ctx_die;
-  clang::DeclContext *decl_ctx =
-      GetClangDeclContextContainingDIE(die, &decl_ctx_die);
+  bool clang_type_was_created = false;
+  clang::DeclContext *decl_ctx = GetClangDeclContextContainingDIE(die, nullptr);
 
-  PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
+  PrepareContextToReceiveMembers(m_ast, GetClangASTImporter(), decl_ctx, die,
                                  attrs.name.GetCString());
 
   if (attrs.accessibility == eAccessNone && decl_ctx) {
@@ -1897,17 +1856,20 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
             tag_decl_kind, template_param_infos);
     clang_type =
         m_ast.CreateClassTemplateSpecializationType(class_specialization_decl);
+    clang_type_was_created = true;
 
     m_ast.SetMetadata(class_template_decl, metadata);
     m_ast.SetMetadata(class_specialization_decl, metadata);
   }
 
-  if (!clang_type) {
+  if (!clang_type_was_created) {
+    clang_type_was_created = true;
     clang_type = m_ast.CreateRecordType(
         decl_ctx, GetOwningClangModule(die), attrs.accessibility,
         attrs.name.GetCString(), tag_decl_kind, attrs.class_language, &metadata,
         attrs.exports_symbols);
   }
+
   // Store a forward declaration to this class type in case any
   // parameters in any class methods need it for the clang types for
   // function prototypes.
@@ -1918,19 +1880,13 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       Type::ResolveState::Forward,
       TypePayloadClang(OptionalClangModuleID(), attrs.is_complete_objc_class));
 
-  // UniqueDWARFASTType is large, so don't create a local variables on the
-  // stack, put it on the heap. This function is often called recursively and
-  // clang isn't good at sharing the stack space for variables in different
-  // blocks.
-  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
   // Add our type to the unique type map so we don't end up creating many
   // copies of the same type over and over in the ASTContext for our
   // module
   unique_ast_entry_up->m_type_sp = type_sp;
   unique_ast_entry_up->m_die = die;
   unique_ast_entry_up->m_declaration = unique_decl;
-  unique_ast_entry_up->m_byte_size = byte_size;
-  unique_ast_entry_up->m_is_forward_declaration = attrs.is_forward_declaration;
+  unique_ast_entry_up->m_byte_size = attrs.byte_size.value_or(0);
   dwarf->GetUniqueDWARFASTTypeMap().Insert(unique_typename,
                                            *unique_ast_entry_up);
 
@@ -1971,7 +1927,7 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
           GetClangASTImporter().SetRecordLayout(record_decl, layout);
         }
       }
-    } else {
+    } else if (clang_type_was_created) {
       // Start the definition if the class is not objective C since the
       // underlying decls respond to isCompleteDefinition(). Objective
       // C decls don't respond to isCompleteDefinition() so we can't
@@ -1983,21 +1939,26 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       if (attrs.class_language != eLanguageTypeObjC &&
           attrs.class_language != eLanguageTypeObjC_plus_plus)
         TypeSystemClang::StartTagDeclarationDefinition(clang_type);
+
+      // Leave this as a forward declaration until we need to know the
+      // details of the type. lldb_private::Type will automatically call
+      // the SymbolFile virtual function
+      // "SymbolFileDWARF::CompleteType(Type *)" When the definition
+      // needs to be defined.
+      assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
+                 ClangUtil::RemoveFastQualifiers(clang_type)
+                     .GetOpaqueQualType()) &&
+             "Type already in the forward declaration map!");
+      // Can't assume m_ast.GetSymbolFile() is actually a
+      // SymbolFileDWARF, it can be a SymbolFileDWARFDebugMap for Apple
+      // binaries.
+      dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
+          ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
+          *die.GetDIERef());
+      m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
     }
   }
 
-  // If this is a declaration DIE, leave this as a forward declaration until we
-  // need to know the details of the type. lldb_private::Type will automatically
-  // call the SymbolFile virtual function "SymbolFileDWARF::CompleteType(Type
-  // *)" When the definition needs to be defined.
-  assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
-             ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType()) &&
-         "Type already in the forward declaration map!");
-  dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
-      ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
-      *die.GetDIERef());
-  m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
-
   // If we made a clang type, set the trivial abi if applicable: We only
   // do this for pass by value - which implies the Trivial ABI. There
   // isn't a way to assert that something that would normally be pass by
@@ -2236,10 +2197,6 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die,
       // For objective C we don't start the definition when the class is
       // created.
       TypeSystemClang::StartTagDeclarationDefinition(clang_type);
-    } else {
-      assert(clang_type.IsBeingDefined() &&
-             "Trying to complete a definition without a prior call to "
-             "StartTagDeclarationDefinition.");
     }
 
     AccessType default_accessibility = eAccessNone;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 853b8ccc30369fd2f4a8c9566c263a0237b1b44b..8d4af203bb2871cbb7f9ac4f7ee475c8fce6dadf 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -42,40 +42,40 @@ struct ParsedDWARFTypeAttributes;
 
 class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
 public:
-  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
-
   DWARFASTParserClang(lldb_private::TypeSystemClang &ast);
 
   ~DWARFASTParserClang() override;
 
   // DWARFASTParser interface.
-  lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die,
-                                  bool *type_is_new_ptr) override;
+  lldb::TypeSP
+  ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die,
+                     bool *type_is_new_ptr) override;
 
-  lldb_private::ConstString
-  ConstructDemangledNameFromDWARF(const DWARFDIE &die) override;
+  lldb_private::ConstString ConstructDemangledNameFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::Function *
   ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit,
-                         const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
                          const lldb_private::AddressRange &func_range) override;
 
   bool
-  CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type,
+  CompleteTypeFromDWARF(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &compiler_type) override;
 
-  lldb_private::CompilerDecl
-  GetDeclForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDecl GetDeclForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   void EnsureAllDIEsInDeclContextHaveBeenParsed(
       lldb_private::CompilerDeclContext decl_context) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextContainingUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::ClangASTImporter &GetClangASTImporter();
 
@@ -105,13 +105,8 @@ public:
   /// \return A string, including surrounding '<>', of the template parameters.
   /// If the DIE's name already has '<>', returns an empty ConstString because
   /// it's assumed that the caller is using the DIE name anyway.
-  lldb_private::ConstString
-  GetDIEClassTemplateParams(const DWARFDIE &die) override;
-
-  // Searching for definition DIE for the given DIE and return the type
-  // associated with the definition DIE, or nullptr if definition DIE is not
-  // found.
-  lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) override;
+  lldb_private::ConstString GetDIEClassTemplateParams(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
 protected:
   /// Protected typedefs and members.
@@ -123,7 +118,8 @@ protected:
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
       clang::DeclContext *>
       DIEToDeclContextMap;
-  typedef std::multimap<const clang::DeclContext *, const DWARFDIE>
+  typedef std::multimap<const clang::DeclContext *,
+                        const lldb_private::plugin::dwarf::DWARFDIE>
       DeclContextToDIEMap;
   typedef llvm::DenseMap<
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
@@ -141,11 +137,14 @@ protected:
   std::unique_ptr<lldb_private::ClangASTImporter> m_clang_ast_importer_up;
   /// @}
 
-  clang::DeclContext *GetDeclContextForBlock(const DWARFDIE &die);
+  clang::DeclContext *
+  GetDeclContextForBlock(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::BlockDecl *ResolveBlockDIE(const DWARFDIE &die);
+  clang::BlockDecl *
+  ResolveBlockDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::NamespaceDecl *ResolveNamespaceDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *
+  ResolveNamespaceDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// Returns the namespace decl that a DW_TAG_imported_declaration imports.
   ///
@@ -156,86 +155,96 @@ protected:
   ///          'die' imports. If the imported entity is not a namespace
   ///          or another import declaration, returns nullptr. If an error
   ///          occurs, returns nullptr.
-  clang::NamespaceDecl *ResolveImportedDeclarationDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *ResolveImportedDeclarationDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool ParseTemplateDIE(const DWARFDIE &die,
+  bool ParseTemplateDIE(const lldb_private::plugin::dwarf::DWARFDIE &die,
                         lldb_private::TypeSystemClang::TemplateParameterInfos
                             &template_param_infos);
 
   bool ParseTemplateParameterInfos(
-      const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       lldb_private::TypeSystemClang::TemplateParameterInfos
           &template_param_infos);
 
-  std::string GetCPlusPlusQualifiedName(const DWARFDIE &die);
+  std::string
+  GetCPlusPlusQualifiedName(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   bool ParseChildMembers(
-      const DWARFDIE &die, lldb_private::CompilerType &class_compiler_type,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::CompilerType &class_compiler_type,
       std::vector<std::unique_ptr<clang::CXXBaseSpecifier>> &base_classes,
-      std::vector<DWARFDIE> &member_function_dies,
-      std::vector<DWARFDIE> &contained_type_dies,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &member_function_dies,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &contained_type_dies,
       DelayedPropertyList &delayed_properties,
       const lldb::AccessType default_accessibility,
       lldb_private::ClangASTImporter::LayoutInfo &layout_info);
 
   size_t
   ParseChildParameters(clang::DeclContext *containing_decl_ctx,
-                       const DWARFDIE &parent_die, bool skip_artificial,
-                       bool &is_static, bool &is_variadic,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                       bool skip_artificial, bool &is_static, bool &is_variadic,
                        bool &has_template_params,
                        std::vector<lldb_private::CompilerType> &function_args,
                        std::vector<clang::ParmVarDecl *> &function_param_decls,
                        unsigned &type_quals);
 
-  size_t ParseChildEnumerators(lldb_private::CompilerType &compiler_type,
-                               bool is_signed, uint32_t enumerator_byte_size,
-                               const DWARFDIE &parent_die);
+  size_t ParseChildEnumerators(
+      lldb_private::CompilerType &compiler_type, bool is_signed,
+      uint32_t enumerator_byte_size,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die);
 
   /// Parse a structure, class, or union type DIE.
-  lldb::TypeSP ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
-                                     const DWARFDIE &die,
-                                     ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
+                        const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        ParsedDWARFTypeAttributes &attrs);
 
-  clang::Decl *GetClangDeclForDIE(const DWARFDIE &die);
+  clang::Decl *
+  GetClangDeclForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *
+  GetClangDeclContextForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextContainingDIE(const DWARFDIE &die,
-                                                       DWARFDIE *decl_ctx_die);
-  lldb_private::OptionalClangModuleID GetOwningClangModule(const DWARFDIE &die);
+  clang::DeclContext *GetClangDeclContextContainingDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::plugin::dwarf::DWARFDIE *decl_ctx_die);
+  lldb_private::OptionalClangModuleID
+  GetOwningClangModule(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool CopyUniqueClassMethodTypes(const DWARFDIE &src_class_die,
-                                  const DWARFDIE &dst_class_die,
-                                  lldb_private::Type *class_type,
-                                  std::vector<DWARFDIE> &failures);
+  bool CopyUniqueClassMethodTypes(
+      const lldb_private::plugin::dwarf::DWARFDIE &src_class_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &dst_class_die,
+      lldb_private::Type *class_type,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &failures);
 
-  clang::DeclContext *GetCachedClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *GetCachedClangDeclContextForDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, const DWARFDIE &die);
+  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx,
+                            const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclToDIE(clang::Decl *decl, const DWARFDIE &die);
+  void LinkDeclToDIE(clang::Decl *decl,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// If \p type_sp is valid, calculate and set its symbol context scope, and
   /// update the type list for its backing symbol file.
   ///
   /// Returns \p type_sp.
-  lldb::TypeSP
-  UpdateSymbolContextScopeForType(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die, lldb::TypeSP type_sp);
+  lldb::TypeSP UpdateSymbolContextScopeForType(
+      const lldb_private::SymbolContext &sc,
+      const lldb_private::plugin::dwarf::DWARFDIE &die, lldb::TypeSP type_sp);
 
   /// Follow Clang Module Skeleton CU references to find a type definition.
-  lldb::TypeSP ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
-                                        const DWARFDIE &die,
-                                        lldb_private::Log *log);
+  lldb::TypeSP
+  ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
+                           const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           lldb_private::Log *log);
 
   // Return true if this type is a declaration to a type in an external
   // module.
-  lldb::ModuleSP GetModuleForType(const DWARFDIE &die);
-
-  void PrepareContextToReceiveMembers(clang::DeclContext *decl_ctx,
-                                      const DWARFDIE &decl_ctx_die,
-                                      const DWARFDIE &die,
-                                      const char *type_name_cstr);
+  lldb::ModuleSP
+  GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   static bool classof(const DWARFASTParser *Parser) {
     return Parser->GetKind() == Kind::DWARFASTParserClang;
@@ -265,8 +274,10 @@ private:
 
   /// Parsed form of all attributes that are relevant for parsing type members.
   struct MemberAttributes {
-    explicit MemberAttributes(const DWARFDIE &die, const DWARFDIE &parent_die,
-                              lldb::ModuleSP module_sp);
+    explicit MemberAttributes(
+        const lldb_private::plugin::dwarf::DWARFDIE &die,
+        const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+        lldb::ModuleSP module_sp);
     const char *name = nullptr;
     /// Indicates how many bits into the word (according to the host endianness)
     /// the low-order bit of the field starts. Can be negative.
@@ -313,12 +324,15 @@ private:
   /// created property.
   /// \param delayed_properties The list of delayed properties that the result
   /// will be appended to.
-  void ParseObjCProperty(const DWARFDIE &die, const DWARFDIE &parent_die,
-                         const lldb_private::CompilerType &class_clang_type,
-                         DelayedPropertyList &delayed_properties);
+  void
+  ParseObjCProperty(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                    const lldb_private::CompilerType &class_clang_type,
+                    DelayedPropertyList &delayed_properties);
 
   void
-  ParseSingleMember(const DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseSingleMember(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                     const lldb_private::CompilerType &class_clang_type,
                     lldb::AccessType default_accessibility,
                     lldb_private::ClangASTImporter::LayoutInfo &layout_info,
@@ -336,25 +350,31 @@ private:
   /// \param[in] class_clang_type The parent RecordType of the static
   ///                             member this function will create.
   void CreateStaticMemberVariable(
-      const DWARFDIE &die, const MemberAttributes &attrs,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      const MemberAttributes &attrs,
       const lldb_private::CompilerType &class_clang_type);
 
-  bool CompleteRecordType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteRecordType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                          lldb_private::Type *type,
                           lldb_private::CompilerType &clang_type);
-  bool CompleteEnumType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteEnumType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &clang_type);
 
-  lldb::TypeSP ParseTypeModifier(const lldb_private::SymbolContext &sc,
-                                 const DWARFDIE &die,
-                                 ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseTypeModifier(const lldb_private::SymbolContext &sc,
+                    const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    ParsedDWARFTypeAttributes &attrs);
   lldb::TypeSP ParseEnum(const lldb_private::SymbolContext &sc,
-                         const DWARFDIE &die, ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseSubroutine(const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
+                         ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseSubroutine(const lldb_private::plugin::dwarf::DWARFDIE &die,
                                const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseArrayType(const DWARFDIE &die,
+  lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die,
-                                        const ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParsePointerToMemberType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           const ParsedDWARFTypeAttributes &attrs);
 
   /// Parses a DW_TAG_inheritance DIE into a base/super class.
   ///
@@ -371,7 +391,8 @@ private:
   /// \param layout_info The layout information that will be updated for C++
   /// base classes with the base offset.
   void ParseInheritance(
-      const DWARFDIE &die, const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       const lldb_private::CompilerType class_clang_type,
       const lldb::AccessType default_accessibility,
       const lldb::ModuleSP &module_sp,
@@ -388,7 +409,8 @@ private:
   /// \param layout_info The layout information that will be updated for
   //   base classes with the base offset
   void
-  ParseRustVariantPart(DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseRustVariantPart(lldb_private::plugin::dwarf::DWARFDIE &die,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                        lldb_private::CompilerType &class_clang_type,
                        const lldb::AccessType default_accesibility,
                        lldb_private::ClangASTImporter::LayoutInfo &layout_info);
@@ -398,9 +420,8 @@ private:
 /// Some attributes are relevant for all kinds of types (declaration), while
 /// others are only meaningful to a specific type (is_virtual)
 struct ParsedDWARFTypeAttributes {
-  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
-
-  explicit ParsedDWARFTypeAttributes(const DWARFDIE &die);
+  explicit ParsedDWARFTypeAttributes(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   lldb::AccessType accessibility = lldb::eAccessNone;
   bool is_artificial = false;
@@ -417,7 +438,7 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
-  DWARFDIE object_pointer;
+  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
   lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
   lldb_private::plugin::dwarf::DWARFFormValue containing_type;
   lldb_private::plugin::dwarf::DWARFFormValue signature;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
index 6330470b970e786eaf535967931666f3fa0d723e..90e42be7202d8779bc44d35789006348dd91ffa8 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
@@ -85,11 +85,6 @@ bool DebugNamesDWARFIndex::ProcessEntry(
   DWARFDIE die = GetDIE(entry);
   if (!die)
     return true;
-  // Clang erroneously emits index entries for declaration DIEs in case when the
-  // definition is in a type unit (llvm.org/pr77696). Weed those out.
-  if (die.IsStructUnionOrClass() &&
-      die.GetAttributeValueAsUnsigned(DW_AT_declaration, 0))
-    return true;
   return callback(die);
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index af3ba2cd5b39d00b8616cae935ad31f4e078dec4..a52a7d6767374283c36e9aa318242576c79e2597 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -481,13 +481,6 @@ static ConstString GetDWARFMachOSegmentName() {
   return g_dwarf_section_name;
 }
 
-llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> &
-SymbolFileDWARF::GetForwardDeclCompilerTypeToDIE() {
-  if (SymbolFileDWARFDebugMap *debug_map_symfile = GetDebugMapSymfile())
-    return debug_map_symfile->GetForwardDeclCompilerTypeToDIE();
-  return m_forward_decl_compiler_type_to_die;
-}
-
 UniqueDWARFASTTypeMap &SymbolFileDWARF::GetUniqueDWARFASTTypeMap() {
   SymbolFileDWARFDebugMap *debug_map_symfile = GetDebugMapSymfile();
   if (debug_map_symfile)
@@ -1639,33 +1632,27 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) {
     return true;
   }
 
-  // Once we start resolving this type, remove it from the forward
-  // declaration map in case anyone's child members or other types require this
-  // type to get resolved.
-  DWARFDIE dwarf_die = GetDIE(die_it->second);
-  GetForwardDeclCompilerTypeToDIE().erase(die_it);
-  Type *type = nullptr;
-  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
-    type = dwarf_ast->FindDefinitionTypeForDIE(dwarf_die);
-  if (!type)
-    return false;
-
-  die_it = GetForwardDeclCompilerTypeToDIE().find(
-      compiler_type_no_qualifiers.GetOpaqueQualType());
-  if (die_it != GetForwardDeclCompilerTypeToDIE().end()) {
-    dwarf_die = GetDIE(die_it->getSecond());
+  DWARFDIE dwarf_die = GetDIE(die_it->getSecond());
+  if (dwarf_die) {
+    // Once we start resolving this type, remove it from the forward
+    // declaration map in case anyone child members or other types require this
+    // type to get resolved. The type will get resolved when all of the calls
+    // to SymbolFileDWARF::ResolveClangOpaqueTypeDefinition are done.
     GetForwardDeclCompilerTypeToDIE().erase(die_it);
-  }
 
-  if (Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion))
-    GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
-        log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
-        dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
-        dwarf_die.Tag(), type->GetName().AsCString());
-  assert(compiler_type);
-  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
-    return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
-  return true;
+    Type *type = GetDIEToType().lookup(dwarf_die.GetDIE());
+
+    Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion);
+    if (log)
+      GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
+          log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
+          dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
+          dwarf_die.Tag(), type->GetName().AsCString());
+    assert(compiler_type);
+    if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
+      return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
+  }
+  return false;
 }
 
 Type *SymbolFileDWARF::ResolveType(const DWARFDIE &die,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 35893f2072dd64817739f50ec374bae3cca8fc61..7282c08c6857c9c000ef98e0eaf8864f53098a59 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -335,8 +335,12 @@ public:
 
   virtual DIEToTypePtr &GetDIEToType() { return m_die_to_type; }
 
-  virtual llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> &
-  GetForwardDeclCompilerTypeToDIE();
+  typedef llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef>
+      CompilerTypeToDIE;
+
+  virtual CompilerTypeToDIE &GetForwardDeclCompilerTypeToDIE() {
+    return m_forward_decl_compiler_type_to_die;
+  }
 
   typedef llvm::DenseMap<const DWARFDebugInfoEntry *, lldb::VariableSP>
       DIEToVariableSP;
@@ -529,14 +533,9 @@ protected:
   NameToOffsetMap m_function_scope_qualified_name_map;
   std::unique_ptr<DWARFDebugRanges> m_ranges;
   UniqueDWARFASTTypeMap m_unique_ast_type_map;
-  // A map from DIE to lldb_private::Type. For record type, the key might be
-  // either declaration DIE or definition DIE.
   DIEToTypePtr m_die_to_type;
   DIEToVariableSP m_die_to_variable_sp;
-  // A map from CompilerType to the struct/class/union/enum DIE (might be a
-  // declaration or a definition) that is used to construct it.
-  llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef>
-      m_forward_decl_compiler_type_to_die;
+  CompilerTypeToDIE m_forward_decl_compiler_type_to_die;
   llvm::DenseMap<dw_offset_t, std::unique_ptr<SupportFileList>>
       m_type_unit_support_files;
   std::vector<uint32_t> m_lldb_cu_to_dwarf_unit;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index d7d571919bc7d6fbd9293cd2af1613fcca217897..de22dd676eef0a1bd4df785d84efbbff2af1470b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -284,11 +284,6 @@ protected:
   lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE(
       const DWARFDIE &die, ConstString type_name, bool must_be_implementation);
 
-  llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> &
-  GetForwardDeclCompilerTypeToDIE() {
-    return m_forward_decl_compiler_type_to_die;
-  }
-
   UniqueDWARFASTTypeMap &GetUniqueDWARFASTTypeMap() {
     return m_unique_ast_type_map;
   }
@@ -326,10 +321,6 @@ protected:
   std::vector<uint32_t> m_func_indexes; // Sorted by address
   std::vector<uint32_t> m_glob_indexes;
   std::map<std::pair<ConstString, llvm::sys::TimePoint<>>, OSOInfoSP> m_oso_map;
-  // A map from CompilerType to the struct/class/union/enum DIE (might be a
-  // declaration or a definition) that is used to construct it.
-  llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef>
-      m_forward_decl_compiler_type_to_die;
   UniqueDWARFASTTypeMap m_unique_ast_type_map;
   LazyBool m_supports_DW_AT_APPLE_objc_complete_type;
   DebugMap m_debug_map;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
index e4db39cabf6feece0cb2877bbb0321fb39d41d87..71c9997e4c82c1ec84781b22a4b2dfdc8ecb1ed6 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
@@ -110,7 +110,7 @@ SymbolFileDWARF::DIEToVariableSP &SymbolFileDWARFDwo::GetDIEToVariable() {
   return GetBaseSymbolFile().GetDIEToVariable();
 }
 
-llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> &
+SymbolFileDWARF::CompilerTypeToDIE &
 SymbolFileDWARFDwo::GetForwardDeclCompilerTypeToDIE() {
   return GetBaseSymbolFile().GetForwardDeclCompilerTypeToDIE();
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
index 2f0ac415e90d402c9b337a59c2033fd1c4208d9f..1500540424b52434f16e2c5b527ed8b1654e22db 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
@@ -72,8 +72,7 @@ protected:
 
   DIEToVariableSP &GetDIEToVariable() override;
 
-  llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> &
-  GetForwardDeclCompilerTypeToDIE() override;
+  CompilerTypeToDIE &GetForwardDeclCompilerTypeToDIE() override;
 
   UniqueDWARFASTTypeMap &GetUniqueDWARFASTTypeMap() override;
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
index 3d201e96f92c3c72271b6ca459696a43d27c725e..223518f0ae824184a01a234d3e2f363c2ed66cab 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
@@ -13,75 +13,66 @@
 using namespace lldb_private::dwarf;
 using namespace lldb_private::plugin::dwarf;
 
-static bool IsStructOrClassTag(llvm::dwarf::Tag Tag) {
-  return Tag == llvm::dwarf::Tag::DW_TAG_class_type ||
-         Tag == llvm::dwarf::Tag::DW_TAG_structure_type;
-}
-
-UniqueDWARFASTType *UniqueDWARFASTTypeList::Find(
-    const DWARFDIE &die, const lldb_private::Declaration &decl,
-    const int32_t byte_size, bool is_forward_declaration) {
-  for (UniqueDWARFASTType &udt : m_collection) {
+bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die,
+                                  const lldb_private::Declaration &decl,
+                                  const int32_t byte_size,
+                                  UniqueDWARFASTType &entry) const {
+  for (const UniqueDWARFASTType &udt : m_collection) {
     // Make sure the tags match
-    if (udt.m_die.Tag() == die.Tag() || (IsStructOrClassTag(udt.m_die.Tag()) &&
-                                         IsStructOrClassTag(die.Tag()))) {
-      // If they are not both definition DIEs or both declaration DIEs, then
-      // don't check for byte size and declaration location, because declaration
-      // DIEs usually don't have those info.
-      bool matching_size_declaration =
-          udt.m_is_forward_declaration != is_forward_declaration
-              ? true
-              : (udt.m_byte_size < 0 || byte_size < 0 ||
-                 udt.m_byte_size == byte_size) &&
-                    udt.m_declaration == decl;
-      if (!matching_size_declaration)
-        continue;
-      // The type has the same name, and was defined on the same file and
-      // line. Now verify all of the parent DIEs match.
-      DWARFDIE parent_arg_die = die.GetParent();
-      DWARFDIE parent_pos_die = udt.m_die.GetParent();
-      bool match = true;
-      bool done = false;
-      while (!done && match && parent_arg_die && parent_pos_die) {
-        const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
-        const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
-        if (parent_arg_tag == parent_pos_tag ||
-            (IsStructOrClassTag(parent_arg_tag) &&
-             IsStructOrClassTag(parent_pos_tag))) {
-          switch (parent_arg_tag) {
-          case DW_TAG_class_type:
-          case DW_TAG_structure_type:
-          case DW_TAG_union_type:
-          case DW_TAG_namespace: {
-            const char *parent_arg_die_name = parent_arg_die.GetName();
-            if (parent_arg_die_name == nullptr) {
-              // Anonymous (i.e. no-name) struct
-              match = false;
-            } else {
-              const char *parent_pos_die_name = parent_pos_die.GetName();
-              if (parent_pos_die_name == nullptr ||
-                  ((parent_arg_die_name != parent_pos_die_name) &&
-                   strcmp(parent_arg_die_name, parent_pos_die_name)))
-                match = false;
+    if (udt.m_die.Tag() == die.Tag()) {
+      // Validate byte sizes of both types only if both are valid.
+      if (udt.m_byte_size < 0 || byte_size < 0 ||
+          udt.m_byte_size == byte_size) {
+        // Make sure the file and line match
+        if (udt.m_declaration == decl) {
+          // The type has the same name, and was defined on the same file and
+          // line. Now verify all of the parent DIEs match.
+          DWARFDIE parent_arg_die = die.GetParent();
+          DWARFDIE parent_pos_die = udt.m_die.GetParent();
+          bool match = true;
+          bool done = false;
+          while (!done && match && parent_arg_die && parent_pos_die) {
+            const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
+            const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
+            if (parent_arg_tag == parent_pos_tag) {
+              switch (parent_arg_tag) {
+              case DW_TAG_class_type:
+              case DW_TAG_structure_type:
+              case DW_TAG_union_type:
+              case DW_TAG_namespace: {
+                const char *parent_arg_die_name = parent_arg_die.GetName();
+                if (parent_arg_die_name ==
+                    nullptr) // Anonymous (i.e. no-name) struct
+                {
+                  match = false;
+                } else {
+                  const char *parent_pos_die_name = parent_pos_die.GetName();
+                  if (parent_pos_die_name == nullptr ||
+                      ((parent_arg_die_name != parent_pos_die_name) &&
+                       strcmp(parent_arg_die_name, parent_pos_die_name)))
+                    match = false;
+                }
+              } break;
+
+              case DW_TAG_compile_unit:
+              case DW_TAG_partial_unit:
+                done = true;
+                break;
+              default:
+                break;
+              }
             }
-          } break;
+            parent_arg_die = parent_arg_die.GetParent();
+            parent_pos_die = parent_pos_die.GetParent();
+          }
 
-          case DW_TAG_compile_unit:
-          case DW_TAG_partial_unit:
-            done = true;
-            break;
-          default:
-            break;
+          if (match) {
+            entry = udt;
+            return true;
           }
         }
-        parent_arg_die = parent_arg_die.GetParent();
-        parent_pos_die = parent_pos_die.GetParent();
-      }
-
-      if (match) {
-        return &udt;
       }
     }
   }
-  return nullptr;
+  return false;
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
index 29e5c02dcbe176a7a3b599f0312c91b96a6aeb4d..bf3cbae55e5c7b939233874fc355f6ec2cc4301d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
@@ -23,19 +23,31 @@ public:
   // Constructors and Destructors
   UniqueDWARFASTType() : m_type_sp(), m_die(), m_declaration() {}
 
+  UniqueDWARFASTType(lldb::TypeSP &type_sp, const DWARFDIE &die,
+                     const Declaration &decl, int32_t byte_size)
+      : m_type_sp(type_sp), m_die(die), m_declaration(decl),
+        m_byte_size(byte_size) {}
+
   UniqueDWARFASTType(const UniqueDWARFASTType &rhs)
       : m_type_sp(rhs.m_type_sp), m_die(rhs.m_die),
-        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size),
-        m_is_forward_declaration(rhs.m_is_forward_declaration) {}
+        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size) {}
 
   ~UniqueDWARFASTType() = default;
 
+  UniqueDWARFASTType &operator=(const UniqueDWARFASTType &rhs) {
+    if (this != &rhs) {
+      m_type_sp = rhs.m_type_sp;
+      m_die = rhs.m_die;
+      m_declaration = rhs.m_declaration;
+      m_byte_size = rhs.m_byte_size;
+    }
+    return *this;
+  }
+
   lldb::TypeSP m_type_sp;
   DWARFDIE m_die;
   Declaration m_declaration;
   int32_t m_byte_size = -1;
-  // True if the m_die is a forward declaration DIE.
-  bool m_is_forward_declaration = true;
 };
 
 class UniqueDWARFASTTypeList {
@@ -50,9 +62,8 @@ public:
     m_collection.push_back(entry);
   }
 
-  UniqueDWARFASTType *Find(const DWARFDIE &die, const Declaration &decl,
-                           const int32_t byte_size,
-                           bool is_forward_declaration);
+  bool Find(const DWARFDIE &die, const Declaration &decl,
+            const int32_t byte_size, UniqueDWARFASTType &entry) const;
 
 protected:
   typedef std::vector<UniqueDWARFASTType> collection;
@@ -69,15 +80,14 @@ public:
     m_collection[name.GetCString()].Append(entry);
   }
 
-  UniqueDWARFASTType *Find(ConstString name, const DWARFDIE &die,
-                           const Declaration &decl, const int32_t byte_size,
-                           bool is_forward_declaration) {
+  bool Find(ConstString name, const DWARFDIE &die, const Declaration &decl,
+            const int32_t byte_size, UniqueDWARFASTType &entry) const {
     const char *unique_name_cstr = name.GetCString();
-    collection::iterator pos = m_collection.find(unique_name_cstr);
+    collection::const_iterator pos = m_collection.find(unique_name_cstr);
     if (pos != m_collection.end()) {
-      return pos->second.Find(die, decl, byte_size, is_forward_declaration);
+      return pos->second.Find(die, decl, byte_size, entry);
     }
-    return nullptr;
+    return false;
   }
 
 protected:
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index be0848573f8123e666183356ba2f71b28350f93b..2a5300012511a85d347ac6a07dd7e010eb24c3c5 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -355,14 +355,14 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   }
   global_stats.try_emplace("targets", std::move(json_targets));
 
+  ConstStringStats const_string_stats;
+  json::Object json_memory{
+      {"strings", const_string_stats.ToJSON()},
+  };
+  global_stats.try_emplace("memory", std::move(json_memory));
   if (!summary_only) {
-    ConstStringStats const_string_stats;
-    json::Object json_memory{
-        {"strings", const_string_stats.ToJSON()},
-    };
     json::Value cmd_stats = debugger.GetCommandInterpreter().GetStatistics();
     global_stats.try_emplace("modules", std::move(json_modules));
-    global_stats.try_emplace("memory", std::move(json_memory));
     global_stats.try_emplace("commands", std::move(cmd_stats));
   }
 
diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp
index 84f282f1de52075dea2fc43c06c67f72158c7b6a..3fe02e0bf4faf18c2571a376cd4872fe602169a7 100644
--- a/lldb/source/Target/ThreadPlanStepOverRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp
@@ -355,7 +355,7 @@ bool ThreadPlanStepOverRange::DoPlanExplainsStop(Event *event_ptr) {
       return_value = NextRangeBreakpointExplainsStop(stop_info_sp);
     } else {
       if (log)
-        log->PutCString("ThreadPlanStepInRange got asked if it explains the "
+        log->PutCString("ThreadPlanStepOverRange got asked if it explains the "
                         "stop for some reason other than step.");
       return_value = false;
     }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/Makefile
index 6914024392cfd0ce4c38a2cdca798a1bd29226f1..3e5da0ecae6697ec8c543a392fe8a4104dd1c4a7 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/Makefile
@@ -1,4 +1,12 @@
 CXX_SOURCES := main.cpp
 CFLAGS_EXTRAS := -std=c++20
 
+ifeq "1" "$(USE_LIBSTDCPP)"
+  CFLAGS_EXTRAS += -DUSE_LIBSTDCPP
+endif
+
+ifeq "1" "$(USE_LIBCPP)"
+  CFLAGS_EXTRAS += -DUSE_LIBCPP
+endif
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/main.cpp
index 8cb81c3bc9f4c4ec835fadd836154a08a9f340ec..aaf76e0d8d99a157f1d4bc4a74d08ccc8db5d585 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/main.cpp
@@ -1,13 +1,27 @@
+#if defined(USE_LIBSTDCPP)
+#include <bits/c++config.h>
+// glibc++ >= 11 and c++20
+#if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE >= 11
 #include <coroutine>
+#define HAS_CPP_COROUTINES 1
+#endif
+#endif
+
+// libc++ always has 'coroutine' feature.
+#if defined(USE_LIBCPP)
+#include <coroutine>
+#define HAS_CPP_COROUTINES 1
+#endif
 
 bool is_implementation_supported() {
-#ifdef _GLIBCXX_RELEASE
-  return _GLIBCXX_RELEASE >= 11;
-#else
+#ifdef HAS_CPP_COROUTINES
   return true;
+#else
+  return false;
 #endif
 }
 
+#ifdef HAS_CPP_COROUTINES
 // `int_generator` is a stripped down, minimal coroutine generator
 // type.
 struct int_generator {
@@ -39,8 +53,11 @@ int_generator my_generator_func() { co_yield 42; }
 // a place to reliably set a breakpoint on.
 void empty_function_so_we_can_set_a_breakpoint() {}
 
+#endif // HAS_CPP_COROUTINES
+
 int main() {
   bool is_supported = is_implementation_supported();
+#ifdef HAS_CPP_COROUTINES
   int_generator gen = my_generator_func();
   std::coroutine_handle<> type_erased_hdl = gen.hdl;
   std::coroutine_handle<int> incorrectly_typed_hdl =
@@ -48,4 +65,8 @@ int main() {
   gen.hdl.resume();                            // Break at initial_suspend
   gen.hdl.resume();                            // Break after co_yield
   empty_function_so_we_can_set_a_breakpoint(); // Break at final_suspend
+  return 0;
+#else
+  return 0; // Break at initial_suspend
+#endif // HAS_CPP_COROUTINES
 }
diff --git a/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
index 851097bdfecf2d560c1a90656b30afee41daaa8a..f06c9ae14bb7af57218a30b4ccd82aa551bb055b 100644
--- a/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
+++ b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
@@ -85,14 +85,15 @@ class TestStatsAPI(TestBase):
         stats_summary.GetAsJSON(stream_summary)
         debug_stats_summary = json.loads(stream_summary.GetData())
         self.assertNotIn("modules", debug_stats_summary)
-        self.assertNotIn("memory", debug_stats_summary)
         self.assertNotIn("commands", debug_stats_summary)
 
         # Summary values should be the same as in full statistics.
-        # Except the parse time on Mac OS X is not deterministic.
+        # The exceptions to this are:
+        # - The parse time on Mac OS X is not deterministic.
+        # - Memory usage may grow over time due to the use of ConstString.
         for key, value in debug_stats_summary.items():
             self.assertIn(key, debug_stats)
-            if key != "targets" and not key.endswith("Time"):
+            if key != "memory" and key != "targets" and not key.endswith("Time"):
                 self.assertEqual(debug_stats[key], value)
 
     def test_command_stats_api(self):
diff --git a/lldb/test/API/lang/c/inlines/Makefile b/lldb/test/API/lang/c/inlines/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f9555f92be8cd214cccf2f3db20c20f0caa0e6f6
--- /dev/null
+++ b/lldb/test/API/lang/c/inlines/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c 
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py b/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py
index 024b9dad9b0fb61aff8d54ba470008f34cff3426..062fd88f7d27213962543491f547ffb03e5ec253 100644
--- a/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py
+++ b/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py
@@ -1,14 +1,60 @@
-from lldbsuite.test import lldbinline
-from lldbsuite.test import decorators
-
-lldbinline.MakeInlineTest(
-    __file__,
-    globals(),
-    [
-        decorators.expectedFailureAll(
-            compiler="clang",
-            compiler_version=["<", "3.5"],
-            bugnumber="llvm.org/pr27845",
+"""Test that inlined argument variables have their correct location in debuginfo"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestRedefinitionsInInlines(TestBase):
+    # https://github.com/llvm/llvm-project/issues/28219
+    @skipIf(compiler="clang", compiler_version=["<", "3.5"])
+    def test(self):
+        self.source = "main.c"
+        self.build()
+        (target, process, thread, bp1) = lldbutil.run_to_source_breakpoint(
+            self, "first breakpoint", lldb.SBFileSpec(self.source, False)
+        )
+
+        bp2 = target.BreakpointCreateBySourceRegex(
+            "second breakpoint", lldb.SBFileSpec(self.source, False)
+        )
+        bp3 = target.BreakpointCreateBySourceRegex(
+            "third breakpoint", lldb.SBFileSpec(self.source, False)
         )
-    ],
-)
+
+        # When called from main(), test2 is passed in the value of 42 in 'b'
+        self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"])
+
+        process.Continue()
+
+        self.assertState(process.GetState(), lldb.eStateStopped)
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint)
+        self.assertIsNotNone(thread)
+        bp_id = thread.GetStopReasonDataAtIndex(0)
+        self.assertEqual(bp_id, bp2.GetID())
+
+        self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"])
+        self.expect("expression c", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["84"])
+
+        process.Continue()
+
+        # Now we're in test1(), and the first thing it does is call test2(24).  "Step in"
+        # and check that we have the value 24 as the argument.
+        self.assertState(process.GetState(), lldb.eStateStopped)
+        thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint)
+        self.assertIsNotNone(thread)
+        bp_id = thread.GetStopReasonDataAtIndex(0)
+        self.assertEqual(bp_id, bp3.GetID())
+
+        frame = thread.GetFrameAtIndex(0)
+        self.assertTrue(frame.IsInlined())
+        self.assertEqual(frame.GetFunctionName(), "test1")
+
+        thread.StepInto()
+
+        frame = thread.GetFrameAtIndex(0)
+        self.assertTrue(frame.IsInlined())
+        self.assertEqual(frame.GetFunctionName(), "test2")
+
+        self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["24"])
diff --git a/lldb/test/API/lang/c/inlines/main.c b/lldb/test/API/lang/c/inlines/main.c
index 8fe49180800bd2fe940ddf607dcd3ae5a7db8fb8..6ecc04dd508fc3900411c2b9e65b0d9c735e199b 100644
--- a/lldb/test/API/lang/c/inlines/main.c
+++ b/lldb/test/API/lang/c/inlines/main.c
@@ -3,23 +3,22 @@
 inline void test1(int) __attribute__ ((always_inline));
 inline void test2(int) __attribute__ ((always_inline));
 
+// Called once from main with b==42 then called from test1 with b==24.
 void test2(int b) {
-    printf("test2(%d)\n", b); //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["42"])
-    {
-      int c = b * 2;
-      printf("c=%d\n", c); //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["42"])
-                           //% self.expect("expression c", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["84"])
-    }
+  printf("test2(%d)\n", b); // first breakpoint
+  {
+    int c = b * 2;
+    printf("c=%d\n", c); // second breakpoint
+  }
 }
 
 void test1(int a) {
     printf("test1(%d)\n",  a);
-    test2(a+1);//% self.runCmd("step")
-               //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["24"])
+    test2(a + 1); // third breakpoint
 }
 
-int main() {
-    test2(42);
-    test1(23);
-    return 0;
+int main(int argc) {
+  test2(42);
+  test1(23);
+  return 0;
 }
diff --git a/lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py b/lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py
index af9749313376656d8e221148d9a4cad95a35c5f3..f677b869d137912c7fdf5a0ac413531a6c300fa6 100644
--- a/lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py
+++ b/lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py
@@ -47,28 +47,66 @@ class CommandRunInterpreterAPICase(TestBase):
         TestBase.setUp(self)
 
         self.stdin_path = self.getBuildArtifact("stdin.txt")
+        self.stdout_path = self.getBuildArtifact("stdout.txt")
+
+    def run_commands_string(
+        self, command_string, options=lldb.SBCommandInterpreterRunOptions()
+    ):
+        """Run the commands in command_string through RunCommandInterpreter.
+        Returns (n_errors, quit_requested, has_crashed, result_string)."""
 
         with open(self.stdin_path, "w") as input_handle:
-            input_handle.write("nonexistingcommand\nquit")
+            input_handle.write(command_string)
 
-        self.dbg.SetInputFile(open(self.stdin_path, "r"))
+        n_errors = 0
+        quit_requested = False
+        has_crashed = False
 
-        # No need to track the output
-        devnull = open(os.devnull, "w")
-        self.dbg.SetOutputFile(devnull)
-        self.dbg.SetErrorFile(devnull)
+        with open(self.stdin_path, "r") as in_fileH, open(
+            self.stdout_path, "w"
+        ) as out_fileH:
+            self.dbg.SetInputFile(in_fileH)
+
+            self.dbg.SetOutputFile(out_fileH)
+            self.dbg.SetErrorFile(out_fileH)
+
+            n_errors, quit_requested, has_crashed = self.dbg.RunCommandInterpreter(
+                True, False, options, 0, False, False
+            )
+
+        result_string = None
+        with open(self.stdout_path, "r") as out_fileH:
+            result_string = out_fileH.read()
+
+        return (n_errors, quit_requested, has_crashed, result_string)
 
     def test_run_session_with_error_and_quit(self):
         """Run non-existing and quit command returns appropriate values"""
 
-        n_errors, quit_requested, has_crashed = self.dbg.RunCommandInterpreter(
-            True, False, lldb.SBCommandInterpreterRunOptions(), 0, False, False
+        n_errors, quit_requested, has_crashed, _ = self.run_commands_string(
+            "nonexistingcommand\nquit\n"
         )
-
         self.assertGreater(n_errors, 0)
         self.assertTrue(quit_requested)
         self.assertFalse(has_crashed)
 
+    def test_allow_repeat(self):
+        """Try auto-repeat of process launch - the command will fail and
+        the auto-repeat will fail because of no auto-repeat."""
+        options = lldb.SBCommandInterpreterRunOptions()
+        options.SetEchoCommands(False)
+        options.SetAllowRepeats(True)
+
+        n_errors, quit_requested, has_crashed, result_str = self.run_commands_string(
+            "process launch\n\n", options
+        )
+        self.assertEqual(n_errors, 2)
+        self.assertFalse(quit_requested)
+        self.assertFalse(has_crashed)
+
+        self.assertIn("invalid target", result_str)
+        self.assertIn("No auto repeat", result_str)
+
 
 class SBCommandInterpreterRunOptionsCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
@@ -86,6 +124,7 @@ class SBCommandInterpreterRunOptionsCase(TestBase):
         self.assertTrue(opts.GetPrintResults())
         self.assertTrue(opts.GetPrintErrors())
         self.assertTrue(opts.GetAddToHistory())
+        self.assertFalse(opts.GetAllowRepeats())
 
         # Invert values
         opts.SetStopOnContinue(not opts.GetStopOnContinue())
@@ -95,6 +134,7 @@ class SBCommandInterpreterRunOptionsCase(TestBase):
         opts.SetPrintResults(not opts.GetPrintResults())
         opts.SetPrintErrors(not opts.GetPrintErrors())
         opts.SetAddToHistory(not opts.GetAddToHistory())
+        opts.SetAllowRepeats(not opts.GetAllowRepeats())
 
         # Check the value changed
         self.assertTrue(opts.GetStopOnContinue())
@@ -104,3 +144,4 @@ class SBCommandInterpreterRunOptionsCase(TestBase):
         self.assertFalse(opts.GetPrintResults())
         self.assertFalse(opts.GetPrintErrors())
         self.assertFalse(opts.GetAddToHistory())
+        self.assertTrue(opts.GetAllowRepeats())
diff --git a/lldb/test/API/tools/lldb-server/TestPtyServer.py b/lldb/test/API/tools/lldb-server/TestPtyServer.py
index 4bfcf70bfa01baf591f290905748849b66dc8fc0..345f68f6d87d3da89d00b7067db2f1f026b83a4d 100644
--- a/lldb/test/API/tools/lldb-server/TestPtyServer.py
+++ b/lldb/test/API/tools/lldb-server/TestPtyServer.py
@@ -7,6 +7,7 @@ from lldbgdbserverutils import *
 import xml.etree.ElementTree as ET
 
 
+@skipIfRemote
 @skipIf(hostoslist=["windows"])
 class PtyServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     def setUp(self):
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test
index c57cefdaf32d267447d478c99cdad7a83b28d37b..9c0510c34ccae0093f93d798d00b5be2a704358d 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test
@@ -1,9 +1,9 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi
 # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \
-# RUN: -o 'crashlog -a -i -t %t.dir/asi %S/Inputs/application_specific_info/asi.txt' \
+# RUN: -o 'crashlog -i -t %t.dir/asi %S/Inputs/application_specific_info/asi.txt' \
 # RUN: -o "thread list" -o "bt all" 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test
index abd1e7c3da53d5de1eebafad4a66bcb55a0dda93..eb1f5f456a2dca0a4d53b5d5227e50fb4f5dd47a 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test
@@ -1,4 +1,4 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \
 # RUN: -o 'crashlog -V' \
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test
index fccd71ce31f733a12ad90293310d1a76460fcb74..684be2846f78df49da8db9e69dad492c0be2763f 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test
@@ -1,4 +1,4 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test
index 6e2826e88aedf1c6272a6a996a6134a16b3e3774..271a4c2aa90f4549d888be69bef5440334dc747e 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test
@@ -1,4 +1,4 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test
index c2f61963ed0cf8e033e94aae77527db6677ca9fd..a17b7ac18a620195fbc17995e49baf2aff099a9f 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test
@@ -1,9 +1,9 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi
 # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \
-# RUN: -o 'crashlog -a -i -t %t.dir/asi %S/Inputs/application_specific_info/leb.txt' \
+# RUN: -o 'crashlog -i -t %t.dir/asi %S/Inputs/application_specific_info/leb.txt' \
 # RUN: -o "thread list" -o "bt all" 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg
index 417069653d68edb859e2afe3a4ae02ca6acf2ff8..b72b2941989317066875f73d846f5a002ba18201 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg
@@ -3,3 +3,8 @@ if 'system-darwin' not in config.available_features:
 
 if 'lldb-repro' in config.available_features:
   config.unsupported = True
+
+config.environment["LLDB_APPLE_DSYMFORUUID_EXECUTABLE"] = ""
+
+# Temporary parallel image loading deadlock workaround
+config.environment["NO_PARALLEL_IMG_LOADING"] = ""
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
index 81e06868eaee79dadbbd8ae253f990769920ea3c..64cd0904371aa568bdbcef912dbc26a853639bc2 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
@@ -1,4 +1,4 @@
-# REQUIRES: python, native && target-aarch64 && system-darwin
+# REQUIRES: python, native && system-darwin
 
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test
diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
deleted file mode 100644
index d253981b498c81bdea59eecd15e162b514add1d5..0000000000000000000000000000000000000000
--- a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
+++ /dev/null
@@ -1,36 +0,0 @@
-# Test definition DIE searching is delayed until complete type is required.
-
-# UNSUPPORTED: system-windows
-
-# RUN: split-file %s %t
-# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -gdwarf -o %t.out
-# RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s
-
-# CHECK: (lldb) p v1
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't2<t1>'
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
-# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't2<t1>' resolving forward declaration...
-# CHECK: (t2<t1>)  {}
-# CHECK: (lldb) p v2
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
-# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't1' resolving forward declaration...
-
-#--- lldb.cmd
-log enable dwarf comp
-p v1
-p v2
-
-#--- main.cpp
-template<typename T>
-struct t2 {
-};
-struct t1;
-t2<t1> v1; // this CU doesn't have definition DIE for t1, but only declaration DIE for it.
-int main() {
-}
-
-#--- t1_def.cpp
-struct t1 { // this CU contains definition DIE for t1.
-  int x;
-};
-t1 v2;
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a8a4d3b8fbd5fed347e50b61f7c28286cfa937f3
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp
@@ -0,0 +1,44 @@
+// Test that we can correctly resolve forward declared types when they only
+// differ in the template arguments of the surrounding context. The reproducer
+// is sensitive to the order of declarations, so we test in both directions.
+
+// REQUIRES: lld
+
+// RUN: %clang --target=x86_64-pc-linux -c %s -o %t-a.o -g -gsimple-template-names -DFILE_A
+// RUN: %clang --target=x86_64-pc-linux -c %s -o %t-b.o -g -gsimple-template-names -DFILE_B
+// RUN: ld.lld %t-a.o %t-b.o -o %t
+// RUN: %lldb %t -o "target variable --ptr-depth 1 --show-types both_a both_b" -o exit | FileCheck %s
+
+// CHECK: (lldb) target variable
+// CHECK-NEXT: (ReferencesBoth<'A'>) both_a = {
+// CHECK-NEXT:   (Outer<'A'>::Inner *) a = 0x{{[0-9A-Fa-f]*}} {}
+// CHECK-NEXT:   (Outer<'A'>::Inner *) b = 0x{{[0-9A-Fa-f]*}} {}
+// CHECK-NEXT: }
+// CHECK-NEXT: (ReferencesBoth<'B'>) both_b = {
+// CHECK-NEXT:   (Outer<'A'>::Inner *) a = 0x{{[0-9A-Fa-f]*}} {}
+// CHECK-NEXT:   (Outer<'B'>::Inner *) b = 0x{{[0-9A-Fa-f]*}} {}
+// CHECK-NEXT: }
+
+template<char C>
+struct Outer {
+  struct Inner {};
+};
+
+template<char C>
+struct ReferencesBoth {
+  Outer<'A'>::Inner *a;
+  Outer<'B'>::Inner *b;
+};
+
+#ifdef FILE_A
+Outer<'A'>::Inner inner_a;
+extern Outer<'B'>::Inner inner_b;
+
+ReferencesBoth<'A'> both_a{&inner_a, &inner_b};
+
+#else
+extern Outer<'A'>::Inner inner_a;
+Outer<'B'>::Inner inner_b;
+
+ReferencesBoth<'B'> both_b{&inner_a, &inner_b};
+#endif
diff --git a/lldb/tools/debugserver/source/JSON.cpp b/lldb/tools/debugserver/source/JSON.cpp
index 315c52aafc9323d3f4a1311ea14dd33803247d96..5453d857cb214cc144095509b356e21ba05331b7 100644
--- a/lldb/tools/debugserver/source/JSON.cpp
+++ b/lldb/tools/debugserver/source/JSON.cpp
@@ -43,7 +43,7 @@ JSONString::JSONString(const std::string &s)
     : JSONValue(JSONValue::Kind::String), m_data(s) {}
 
 void JSONString::Write(std::ostream &s) {
-  s << "\"" << json_string_quote_metachars(m_data).c_str() << "\"";
+  s << "\"" << json_string_quote_metachars(m_data) << "\"";
 }
 
 uint64_t JSONNumber::GetAsUnsigned() const {
@@ -395,7 +395,7 @@ JSONParser::Token JSONParser::GetToken(std::string &value) {
           } else {
             error << "error: got exponent character but no exponent digits at "
                      "offset in float value \""
-                  << value.c_str() << "\"";
+                  << value << "\"";
             value = error.str();
             return Token::Status;
           }
@@ -405,8 +405,7 @@ JSONParser::Token JSONParser::GetToken(std::string &value) {
           if (got_frac_digits) {
             return Token::Float;
           } else {
-            error << "error: no digits after decimal point \"" << value.c_str()
-                  << "\"";
+            error << "error: no digits after decimal point \"" << value << "\"";
             value = error.str();
             return Token::Status;
           }
@@ -417,7 +416,7 @@ JSONParser::Token JSONParser::GetToken(std::string &value) {
           // We need at least some integer digits to make an integer
           return Token::Integer;
         } else {
-          error << "error: no digits negate sign \"" << value.c_str() << "\"";
+          error << "error: no digits negate sign \"" << value << "\"";
           value = error.str();
           return Token::Status;
         }
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 64898ab09772f4b53e1a22c63c6f33827a732055..3208147101c0d7eba276e0c54ad8c4a018db3af1 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -903,8 +903,8 @@ set(LLVM_PROFDATA_FILE "" CACHE FILEPATH
   "Profiling data file to use when compiling in order to improve runtime performance.")
 
 if(LLVM_INCLUDE_TESTS)
-  # Lit test suite requires at least python 3.6
-  set(LLVM_MINIMUM_PYTHON_VERSION 3.6)
+  # All LLVM Python files should be compatible down to this minimum version.
+  set(LLVM_MINIMUM_PYTHON_VERSION 3.8)
 else()
   # FIXME: it is unknown if this is the actual minimum bound
   set(LLVM_MINIMUM_PYTHON_VERSION 3.0)
diff --git a/llvm/bindings/ocaml/llvm/llvm.ml b/llvm/bindings/ocaml/llvm/llvm.ml
index 7eeaae49e10590489b1db5424a5690276b5f9820..7bfaf8656ab3f2abc5704a0dece9435e3b580ca3 100644
--- a/llvm/bindings/ocaml/llvm/llvm.ml
+++ b/llvm/bindings/ocaml/llvm/llvm.ml
@@ -1135,6 +1135,9 @@ external delete_instruction : llvalue -> unit = "llvm_delete_instruction"
 external builder : llcontext -> llbuilder = "llvm_builder"
 external position_builder : (llbasicblock, llvalue) llpos -> llbuilder -> unit
                           = "llvm_position_builder"
+external position_builder_before_dbg_records : (llbasicblock, llvalue) llpos ->
+                                               llbuilder -> unit
+                                  = "llvm_position_builder_before_dbg_records"
 external insertion_block : llbuilder -> llbasicblock = "llvm_insertion_block"
 external insert_into_builder : llvalue -> string -> llbuilder -> unit
                              = "llvm_insert_into_builder"
@@ -1148,6 +1151,8 @@ let builder_before context i = builder_at context (Before i)
 let builder_at_end context bb = builder_at context (At_end bb)
 
 let position_before i = position_builder (Before i)
+let position_before_dbg_records i =
+  position_builder_before_dbg_records (Before i)
 let position_at_end bb = position_builder (At_end bb)
 
 
diff --git a/llvm/bindings/ocaml/llvm/llvm.mli b/llvm/bindings/ocaml/llvm/llvm.mli
index 36cc095dc66e3de66f9c6dc083c230a4c031d720..89b894b35849da81354c367a217f00e60e1c7b77 100644
--- a/llvm/bindings/ocaml/llvm/llvm.mli
+++ b/llvm/bindings/ocaml/llvm/llvm.mli
@@ -1874,10 +1874,22 @@ val builder_at_end : llcontext -> llbasicblock -> llbuilder
     See the constructor for [llvm::LLVMBuilder]. *)
 val position_builder : (llbasicblock, llvalue) llpos -> llbuilder -> unit
 
+(** [position_builder_before_dbg_records ip bb before_dbg_records] moves the
+    instruction builder [bb] to the position [ip], before any debug records
+    there.
+    See the constructor for [llvm::LLVMBuilder]. *)
+val position_builder_before_dbg_records : (llbasicblock, llvalue) llpos ->
+                                          llbuilder -> unit
+
 (** [position_before ins b] moves the instruction builder [b] to before the
     instruction [isn]. See the method [llvm::LLVMBuilder::SetInsertPoint]. *)
 val position_before : llvalue -> llbuilder -> unit
 
+(** [position_before_dbg_records ins b] moves the instruction builder [b]
+    to before the instruction [isn] and any debug records attached to it.
+    See the method [llvm::LLVMBuilder::SetInsertPoint]. *)
+val position_before_dbg_records : llvalue -> llbuilder -> unit
+
 (** [position_at_end bb b] moves the instruction builder [b] to the end of the
     basic block [bb]. See the method [llvm::LLVMBuilder::SetInsertPoint]. *)
 val position_at_end : llbasicblock -> llbuilder -> unit
diff --git a/llvm/bindings/ocaml/llvm/llvm_ocaml.c b/llvm/bindings/ocaml/llvm/llvm_ocaml.c
index 26a3ac21501d236df204f64a89f8c75149b7c638..3976a96a6da9b4301e6c8e3cc79fcbf1675ba4af 100644
--- a/llvm/bindings/ocaml/llvm/llvm_ocaml.c
+++ b/llvm/bindings/ocaml/llvm/llvm_ocaml.c
@@ -2004,6 +2004,18 @@ value llvm_builder(value C) {
   return alloc_builder(LLVMCreateBuilderInContext(Context_val(C)));
 }
 
+/* (llbasicblock, llvalue) llpos -> llbuilder -> unit */
+value llvm_position_builder_before_dbg_records(value Pos, value B) {
+  if (Tag_val(Pos) == 0) {
+    LLVMBasicBlockRef BB = BasicBlock_val(Field(Pos, 0));
+    LLVMPositionBuilderAtEnd(Builder_val(B), BB);
+  } else {
+    LLVMValueRef I = Value_val(Field(Pos, 0));
+    LLVMPositionBuilderBeforeInstrAndDbgRecords(Builder_val(B), I);
+  }
+  return Val_unit;
+}
+
 /* (llbasicblock, llvalue) llpos -> llbuilder -> unit */
 value llvm_position_builder(value Pos, value B) {
   if (Tag_val(Pos) == 0) {
diff --git a/llvm/cmake/config.guess b/llvm/cmake/config.guess
index f4896236776942eee5943094a74cc7b5301eb770..2444ed7f5792b467923aaff30f62548b9039f815 100644
--- a/llvm/cmake/config.guess
+++ b/llvm/cmake/config.guess
@@ -4,7 +4,7 @@
 #   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
 #   2011 Free Software Foundation, Inc.
 
-timestamp='2011-08-20'
+timestamp='2024-06-07'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -1028,7 +1028,11 @@ EOF
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
-	echo ${UNAME_MACHINE}-ibm-linux
+	if [ "$(grep -Ei 'debian|ubuntu' /etc/lsb-release)" ]; then
+	    echo ${UNAME_MACHINE}-linux-gnu
+	else
+	    echo ${UNAME_MACHINE}-ibm-linux
+	fi
 	exit ;;
     sh64*:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-gnu
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index bb6751038fc9c30c599e6210776e28da27225bb8..aa50ce329d1dea36d19a9e44cd8f0e7ee1641dc4 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -504,6 +504,13 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
+     ``gfx1152``                 ``amdgcn``   APU   - cumode          - Architected                   *TBA*
+                                                    - wavefrontsize64   flat
+                                                                        scratch                       .. TODO::
+                                                                      - Packed
+                                                                        work-item                       Add product
+                                                                        IDs                             names.
+
      ``gfx1200``                 ``amdgcn``   dGPU  - cumode          - Architected                   *TBA*
                                                     - wavefrontsize64   flat
                                                                         scratch                       .. TODO::
@@ -591,11 +598,13 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor
                                          - ``gfx1102``                        - Packed          hazards specific to some targets
                                          - ``gfx1103``                          work-item       within this family.
                                          - ``gfx1150``                          IDs
-                                         - ``gfx1151``                                          Not all VGPRs can be used on:
+                                         - ``gfx1151``
+                                         - ``gfx1152``                                          Not all VGPRs can be used on:
 
                                                                                                 - ``gfx1100``
                                                                                                 - ``gfx1101``
                                                                                                 - ``gfx1151``
+                                                                                                - ``gfx1152``
 
                                                                                                 SALU floating point instructions
                                                                                                 and single-use VGPR hint
@@ -604,12 +613,14 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor
 
                                                                                                 - ``gfx1150``
                                                                                                 - ``gfx1151``
+                                                                                                - ``gfx1152``
 
                                                                                                 SGPRs are not supported for src1
                                                                                                 in dpp instructions for:
 
                                                                                                 - ``gfx1150``
                                                                                                 - ``gfx1151``
+                                                                                                - ``gfx1152``
 
 
      ``gfx12-generic``    ``amdgcn``     - ``gfx1200``     - wavefrontsize64  - Architected     No restrictions.
@@ -1979,7 +1990,7 @@ The AMDGPU backend uses the following ELF header:
      ``EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC``  0x052      ``gfx10-1-generic``
      ``EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC``  0x053      ``gfx10-3-generic``
      ``EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC``    0x054      ``gfx11-generic``
-     *reserved*                                 0x055      Reserved.
+     ``EF_AMDGPU_MACH_AMDGCN_GFX1152``          0x055      ``gfx1152``.
      *reserved*                                 0x056      Reserved.
      *reserved*                                 0x057      Reserved.
      *reserved*                                 0x058      Reserved.
diff --git a/llvm/docs/Benchmarking.rst b/llvm/docs/Benchmarking.rst
index 7deb221446067f56bbb36c5d92f96d6e3ba0563e..cd7b835e47c53582932e1da8ba00030f3140cd9e 100644
--- a/llvm/docs/Benchmarking.rst
+++ b/llvm/docs/Benchmarking.rst
@@ -49,7 +49,7 @@ Linux
 
    for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
    do
-     echo performance > /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+     echo performance > $i
    done
 
 * Use https://github.com/lpechacek/cpuset to reserve cpus for just the
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 7ecef78c405b6f2e6bad6bd2e7a37331477f919c..0a1913dca8aac1631231855fb97c8cc7e4d7e608 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -292,16 +292,16 @@ uses the package and provides other details.
 Package                                                     Version      Notes
 =========================================================== ============ ==========================================
 `CMake <http://cmake.org/>`__                               >=3.20.0     Makefile/workspace generator
-`python <http://www.python.org/>`_                          >=3.6        Automated test suite\ :sup:`1`
+`python <http://www.python.org/>`_                          >=3.8        Automated test suite\ :sup:`1`
 `zlib <http://zlib.net>`_                                   >=1.2.3.4    Compression library\ :sup:`2`
 `GNU Make <http://savannah.gnu.org/projects/make>`_         3.79, 3.79.1 Makefile/build processor\ :sup:`3`
 =========================================================== ============ ==========================================
 
 .. note::
 
-   #. Only needed if you want to run the automated test suite. Python 3.8.0
-      or later is needed on Windows if a substitute (virtual) drive is used
-      to access LLVM source code due to ``MAX_PATH`` limitations.
+   #. Only needed if you want to run the automated test suite in the
+      ``llvm/test`` directory, or if you plan to utilize any Python libraries,
+      utilities, or bindings.
    #. Optional, adds compression / uncompression capabilities to selected LLVM
       tools.
    #. Optional, you can use any other build tool supported by CMake.
diff --git a/llvm/docs/GettingStartedVS.rst b/llvm/docs/GettingStartedVS.rst
index a1eb88dccc9e5c761d8b3437ec903fc7c27c914e..4b15272635fbeb96b2f81a4068ba90f932030454 100644
--- a/llvm/docs/GettingStartedVS.rst
+++ b/llvm/docs/GettingStartedVS.rst
@@ -55,7 +55,7 @@ Visual Studio 2019 so separate installation is not required. If you do install
 CMake separately, Visual Studio 2022 will require CMake Version 3.21 or later.
 
 If you would like to run the LLVM tests you will need `Python
-<http://www.python.org/>`_. Version 3.6 and newer are known to work. You can
+<http://www.python.org/>`_. Version 3.8 and newer are known to work. You can
 install Python with Visual Studio 2019, from the Microsoft store or from
 the `Python web site <http://www.python.org/>`_. We recommend the latter since it
 allows you to adjust installation options.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 9d7ade8eb523b843f4e94eed4c3eeda93d9fb433..9fb2c048a5c8694dc5389cbdb354d4b24e996327 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1924,7 +1924,11 @@ example:
     even if this attribute says the frame pointer can be eliminated.
     The allowed string values are:
 
-     * ``"none"`` (default) - the frame pointer can be eliminated.
+     * ``"none"`` (default) - the frame pointer can be eliminated, and it's
+       register can be used for other purposes.
+     * ``"reserved"`` - the frame pointer register must either be updated to
+       point to a valid frame record for the current function, or not be
+       modified.
      * ``"non-leaf"`` - the frame pointer should be kept if the function calls
        other functions.
      * ``"all"`` - the frame pointer should be kept.
@@ -6308,6 +6312,15 @@ The current supported opcode vocabulary is limited:
   (``16`` and ``DW_ATE_signed`` here, respectively) to which the top of the
   expression stack is to be converted. Maps into a ``DW_OP_convert`` operation
   that references a base type constructed from the supplied values.
+- ``DW_OP_LLVM_extract_bits_sext, 16, 8,`` specifies the offset and size
+  (``16`` and ``8`` here, respectively) of bits that are to be extracted and
+  sign-extended from the value at the top of the expression stack. If the top of
+  the expression stack is a memory location then these bits are extracted from
+  the value pointed to by that memory location. Maps into a ``DW_OP_shl``
+  followed by ``DW_OP_shra``.
+- ``DW_OP_LLVM_extract_bits_zext`` behaves similarly to
+  ``DW_OP_LLVM_extract_bits_sext``, but zero-extends instead of sign-extending.
+  Maps into a ``DW_OP_shl`` followed by ``DW_OP_shr``.
 - ``DW_OP_LLVM_tag_offset, tag_offset`` specifies that a memory tag should be
   optionally applied to the pointer. The memory tag is derived from the
   given tag offset in an implementation-defined manner.
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 393b97efa54705084b1ef6baf6119fb83368268f..8cdb9db087c778194183d2dedeceb124d1ada4bd 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -47,6 +47,11 @@ Non-comprehensive list of changes in this release
 Update on required toolchains to build LLVM
 -------------------------------------------
 
+* The minimum Python version has been raised from 3.6 to 3.8 across all of LLVM.
+  This enables the use of many new Python features, aligning more closely with
+  modern Python best practices, and improves CI maintainability
+  See `#78828 <https://github.com/llvm/llvm-project/pull/78828>`_ for more info.
+
 Changes to the LLVM IR
 ----------------------
 
@@ -101,6 +106,7 @@ Changes to the AMDGPU Backend
 Changes to the ARM Backend
 --------------------------
 
+* Added support for Cortex-R52+ CPU.
 * FEAT_F32MM is no longer activated by default when using `+sve` on v8.6-A or greater. The feature is still available and can be used by adding `+f32mm` to the command line options.
 * armv8-r now implies only fp-armv8d16sp, rather than neon and full fp-armv8. These features are still included by default for cortex-r52. The default cpu for armv8-r is now "generic", for compatibility with variants that do not include neon, fp64, and d32.
 
@@ -116,6 +122,10 @@ Changes to the Hexagon Backend
 Changes to the LoongArch Backend
 --------------------------------
 
+* i32 is now a native type in the datalayout string. This enables
+  LoopStrengthReduce for loops with i32 induction variables, among other
+  optimizations.
+
 Changes to the MIPS Backend
 ---------------------------
 
@@ -204,6 +214,8 @@ Changes to the C API
   * ``LLVMConstICmp``
   * ``LLVMConstFCmp``
 
+* Added ``LLVMPositionBuilderBeforeDbgRecords`` and ``LLVMPositionBuilderBeforeInstrAndDbgRecords``. Same as ``LLVMPositionBuilder`` and ``LLVMPositionBuilderBefore`` except the insertion position is set to before the debug records that precede the target instruction. See the `debug info migration guide <https://llvm.org/docs/RemoveDIsDebugInfo.html>`_ for more info. ``LLVMPositionBuilder`` and ``LLVMPositionBuilderBefore`` are unchanged; they insert before the indicated instruction but after any attached debug records.
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
@@ -213,6 +225,13 @@ Changes to the Metadata Info
 Changes to the Debug Info
 ---------------------------------
 
+* LLVM has switched from using debug intrinsics internally to using debug
+  records by default. This should happen transparently when using the DIBuilder
+  to construct debug variable information, but will require changes for any code
+  that interacts with debug intrinsics directly. Debug intrinsics will only be
+  supported on a best-effort basis from here onwards; for more information, see
+  the `migration docs <https://llvm.org/docs/RemoveDIsDebugInfo.html>`_.
+
 Changes to the LLVM tools
 ---------------------------------
 * llvm-nm and llvm-objdump can now print symbol information from linked
diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md
index 3cdf63c63b5a0bb035d4a66dacacabf2d3ab5a21..ef11c8ed8b43359b321da5954afa6eca207a48a2 100644
--- a/llvm/docs/RemoveDIsDebugInfo.md
+++ b/llvm/docs/RemoveDIsDebugInfo.md
@@ -36,7 +36,7 @@ For a more in-depth overview of how to update existing code to support debug rec
 
 # C-API changes
 
-All the functions that have been added are temporary and will be deprecated in the future. The intention is that they'll help downstream projects adapt during the transition period.
+Some new functions that have been added are temporary and will be deprecated in the future. The intention is that they'll help downstream projects adapt during the transition period.
 
 ```
 New functions (all to be deprecated)
@@ -60,8 +60,20 @@ LLVMDIBuilderInsertDeclareBefore   # Insert a debug record (new debug info forma
 LLVMDIBuilderInsertDeclareAtEnd    # Same as above.
 LLVMDIBuilderInsertDbgValueBefore  # Same as above.
 LLVMDIBuilderInsertDbgValueAtEnd   # Same as above.
+
+New functions (no plans to deprecate)
+----------------------------------
+LLVMPositionBuilderBeforeDbgRecords          # See info below.
+LLVMPositionBuilderBeforeInstrAndDbgRecords  # See info below.
 ```
 
+`LLVMPositionBuilderBeforeDbgRecords` and `LLVMPositionBuilderBeforeInstrAndDbgRecords` behave the same as `LLVMPositionBuilder` and `LLVMPositionBuilderBefore` except the insertion position is set before the debug records that precede the target instruction. Note that this doesn't mean that debug intrinsics before the chosen instruction are skipped, only debug records (which unlike debug records are not themselves instructions).
+
+If you don't know which function to call then follow this rule:
+If you are trying to insert at the start of a block, or purposfully skip debug intrinsics to determine the insertion point for any other reason, then call the new functions.
+
+`LLVMPositionBuilder` and `LLVMPositionBuilderBefore` are unchanged. They insert before the indicated instruction but after any attached debug records.
+
 # The new "Debug Record" model
 
 Below is a brief overview of the new representation that replaces debug intrinsics; for an instructive guide on updating old code, see [here](#how-to-update-existing-code).
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index de27f6b2372db621d3a7a086e0102d05cd991136..70865b95cb3932fddb0d95bb207a9fbbc9ef3b8f 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -17,9 +17,9 @@ in  `the official SPIR-V specification <https://www.khronos.org/registry/SPIR-V/
 Usage
 =====
 
-The SPIR-V backend can be invoked either from LLVM's Static Compiler (llc) or Clang, 
-allowing developers to compile LLVM intermediate language (IL) files or OpenCL kernel 
-sources directly to SPIR-V. This section outlines the usage of various commands to 
+The SPIR-V backend can be invoked either from LLVM's Static Compiler (llc) or Clang,
+allowing developers to compile LLVM intermediate language (IL) files or OpenCL kernel
+sources directly to SPIR-V. This section outlines the usage of various commands to
 leverage the SPIR-V backend for different purposes.
 
 Static Compiler Commands
@@ -89,18 +89,22 @@ to specify the target triple:
      Vendor                Description
      ===================== ==============================================================
      *<empty>*/``unknown``  Generic SPIR-V target without any vendor-specific settings.
+     ``amd``                AMDGCN SPIR-V target, with support for target specific
+                            builtins and ASM, meant to be consumed by AMDGCN toolchains.
      ===================== ==============================================================
 
   .. table:: Operating Systems
 
-     ===================== ============================================================
+     ===================== ==============================================================
      OS                    Description
-     ===================== ============================================================
+     ===================== ==============================================================
      *<empty>*/``unknown``  Defaults to the OpenCL runtime.
      ``vulkan``             Vulkan shader runtime.
      ``vulkan1.2``          Vulkan 1.2 runtime, corresponding to SPIR-V 1.5.
      ``vulkan1.3``          Vulkan 1.3 runtime, corresponding to SPIR-V 1.6.
-     ===================== ============================================================
+     ``amdhsa``             AMDHSA runtime, meant to be used on HSA compatible runtimes,
+                            corresponding to SPIR-V 1.6.
+     ===================== ==============================================================
 
   .. table:: SPIR-V Environments
 
@@ -114,15 +118,17 @@ Example:
 
 ``-target spirv64v1.0`` can be used to compile for SPIR-V version 1.0 with 64-bit pointer width.
 
+``-target spirv64-amd-amdhsa`` can be used to compile for AMDGCN flavoured SPIR-V with 64-bit pointer width.
+
 .. _spirv-extensions:
 
 Extensions
 ----------
 
-The SPIR-V backend supports a variety of `extensions <https://github.com/KhronosGroup/SPIRV-Registry/tree/main/extensions>`_ 
-that enable or enhance features beyond the core SPIR-V specification. 
-These extensions can be enabled using the ``-spirv-extensions`` option 
-followed by the name of the extension(s) you wish to enable. Below is a 
+The SPIR-V backend supports a variety of `extensions <https://github.com/KhronosGroup/SPIRV-Registry/tree/main/extensions>`_
+that enable or enhance features beyond the core SPIR-V specification.
+These extensions can be enabled using the ``-spirv-extensions`` option
+followed by the name of the extension(s) you wish to enable. Below is a
 list of supported SPIR-V extensions, sorted alphabetically by their extension names:
 
 .. list-table:: Supported SPIR-V Extensions
@@ -189,14 +195,14 @@ To enable all extensions except specified, specify ``all`` followed by a list of
 SPIR-V representation in LLVM IR
 ================================
 
-SPIR-V is intentionally designed for seamless integration with various Intermediate 
-Representations (IRs), including LLVM IR, facilitating straightforward mappings for 
-most of its entities. The development of the SPIR-V backend has been guided by a 
+SPIR-V is intentionally designed for seamless integration with various Intermediate
+Representations (IRs), including LLVM IR, facilitating straightforward mappings for
+most of its entities. The development of the SPIR-V backend has been guided by a
 principle of compatibility with the `Khronos Group SPIR-V LLVM Translator <https://github.com/KhronosGroup/SPIRV-LLVM-Translator>`_.
-Consequently, the input representation accepted by the SPIR-V backend aligns closely 
-with that detailed in `the SPIR-V Representation in LLVM document <https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/main/docs/SPIRVRepresentationInLLVM.rst>`_. 
-This document, along with the sections that follow, delineate the main points and focus 
-on any differences between the LLVM IR that this backend processes and the conventions 
+Consequently, the input representation accepted by the SPIR-V backend aligns closely
+with that detailed in `the SPIR-V Representation in LLVM document <https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/main/docs/SPIRVRepresentationInLLVM.rst>`_.
+This document, along with the sections that follow, delineate the main points and focus
+on any differences between the LLVM IR that this backend processes and the conventions
 used by other tools.
 
 .. _spirv-special-types:
@@ -237,10 +243,10 @@ previous type has the representation
 Target Intrinsics
 -----------------
 
-The SPIR-V backend employs several LLVM IR intrinsics that facilitate various low-level 
-operations essential for generating correct and efficient SPIR-V code. These intrinsics 
-cover a range of functionalities from type assignment and memory management to control 
-flow and atomic operations. Below is a detailed table of selected intrinsics used in the 
+The SPIR-V backend employs several LLVM IR intrinsics that facilitate various low-level
+operations essential for generating correct and efficient SPIR-V code. These intrinsics
+cover a range of functionalities from type assignment and memory management to control
+flow and atomic operations. Below is a detailed table of selected intrinsics used in the
 SPIR-V backend, along with their descriptions and argument details.
 
 .. list-table:: LLVM IR Intrinsics for SPIR-V
@@ -369,80 +375,80 @@ SPIR-V backend, along with their descriptions and argument details.
 Builtin Functions
 -----------------
 
-The following section highlights the representation of SPIR-V builtins in LLVM IR, 
+The following section highlights the representation of SPIR-V builtins in LLVM IR,
 emphasizing builtins that do not have direct counterparts in LLVM.
 
 Instructions as Function Calls
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-SPIR-V builtins without direct LLVM counterparts are represented as LLVM function calls. 
-These functions, termed SPIR-V builtin functions, follow an IA64 mangling scheme with 
-SPIR-V-specific extensions. Parsing non-mangled calls to builtins is supported in some cases, 
+SPIR-V builtins without direct LLVM counterparts are represented as LLVM function calls.
+These functions, termed SPIR-V builtin functions, follow an IA64 mangling scheme with
+SPIR-V-specific extensions. Parsing non-mangled calls to builtins is supported in some cases,
 but not tested extensively. The general format is:
 
 .. code-block:: c
 
   __spirv_{OpCodeName}{_OptionalPostfixes}
 
-Where `{OpCodeName}` is the SPIR-V opcode name sans the "Op" prefix, and 
-`{OptionalPostfixes}` are decoration-specific postfixes, if any. The mangling and 
-postfixes allow for the representation of SPIR-V's rich instruction set within LLVM's 
+Where `{OpCodeName}` is the SPIR-V opcode name sans the "Op" prefix, and
+`{OptionalPostfixes}` are decoration-specific postfixes, if any. The mangling and
+postfixes allow for the representation of SPIR-V's rich instruction set within LLVM's
 framework.
 
 Extended Instruction Sets
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-SPIR-V defines several extended instruction sets for additional functionalities, such as 
-OpenCL-specific operations. In LLVM IR, these are represented by function calls to 
+SPIR-V defines several extended instruction sets for additional functionalities, such as
+OpenCL-specific operations. In LLVM IR, these are represented by function calls to
 mangled builtins and selected based on the environment. For example:
 
 .. code-block:: c
 
   acos_f32
 
-represents the `acos` function from the OpenCL extended instruction set for a float32 
+represents the `acos` function from the OpenCL extended instruction set for a float32
 input.
 
 Builtin Variables
 ~~~~~~~~~~~~~~~~~
 
-SPIR-V builtin variables, which provide access to special hardware or execution model 
-properties, are mapped to either LLVM function calls or LLVM global variables. The 
+SPIR-V builtin variables, which provide access to special hardware or execution model
+properties, are mapped to either LLVM function calls or LLVM global variables. The
 representation follows the naming convention:
 
 .. code-block:: c
 
   __spirv_BuiltIn{VariableName}
 
-For instance, the SPIR-V builtin `GlobalInvocationId` is accessible in LLVM IR as 
+For instance, the SPIR-V builtin `GlobalInvocationId` is accessible in LLVM IR as
 `__spirv_BuiltInGlobalInvocationId`.
 
 Vector Load and Store Builtins
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-SPIR-V's capabilities for loading and storing vectors are represented in LLVM IR using 
-functions that mimic the SPIR-V instructions. These builtins handle cases that LLVM's 
-native instructions do not directly support, enabling fine-grained control over memory 
+SPIR-V's capabilities for loading and storing vectors are represented in LLVM IR using
+functions that mimic the SPIR-V instructions. These builtins handle cases that LLVM's
+native instructions do not directly support, enabling fine-grained control over memory
 operations.
 
 Atomic Operations
 ~~~~~~~~~~~~~~~~~
 
-SPIR-V's atomic operations, especially those operating on floating-point data, are 
-represented in LLVM IR with corresponding function calls. These builtins ensure 
-atomicity in operations where LLVM might not have direct support, essential for parallel 
+SPIR-V's atomic operations, especially those operating on floating-point data, are
+represented in LLVM IR with corresponding function calls. These builtins ensure
+atomicity in operations where LLVM might not have direct support, essential for parallel
 execution and synchronization.
 
 Image Operations
 ~~~~~~~~~~~~~~~~
 
-SPIR-V provides extensive support for image and sampler operations, which LLVM 
-represents through function calls to builtins. These include image reads, writes, and 
+SPIR-V provides extensive support for image and sampler operations, which LLVM
+represents through function calls to builtins. These include image reads, writes, and
 queries, allowing detailed manipulation of image data and parameters.
 
 Group and Subgroup Operations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-For workgroup and subgroup operations, LLVM uses function calls to represent SPIR-V's 
-group-based instructions. These builtins facilitate group synchronization, data sharing, 
+For workgroup and subgroup operations, LLVM uses function calls to represent SPIR-V's
+group-based instructions. These builtins facilitate group synchronization, data sharing,
 and collective operations essential for efficient parallel computation.
diff --git a/llvm/docs/TableGen/BackGuide.rst b/llvm/docs/TableGen/BackGuide.rst
index e1413c1c73a795f73e666e23e245c00c35bbc0d7..60677a6dcd627d3b7e88f170d49a7cc8c0553dac 100644
--- a/llvm/docs/TableGen/BackGuide.rst
+++ b/llvm/docs/TableGen/BackGuide.rst
@@ -761,7 +761,7 @@ over time. The output looks like this.
 
   -------------------- Global Variables (5) --------------------
 
-  AMDGPUBufferIntrinsics = [int_amdgcn_buffer_load_format, ...
+  AMDGPUBufferIntrinsics = [int_amdgcn_s_buffer_load, ...
   AMDGPUImageDimAtomicIntrinsics = [int_amdgcn_image_atomic_swap_1d, ...
   ...
   -------------------- Classes (758) --------------------
diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index e24feb3bf5fa2476b33eac04e382bd6cbd19af46..89499d8b937d5c780bad89c4b470d26d5e35ac39 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -23,7 +23,7 @@ Requirements
 ============
 
 In order to use the LLVM testing infrastructure, you will need all of the
-software required to build LLVM, as well as `Python <http://python.org>`_ 3.6 or
+software required to build LLVM, as well as `Python <http://python.org>`_ 3.8 or
 later.
 
 LLVM Testing Infrastructure Organization
diff --git a/llvm/examples/BrainF/BrainF.cpp b/llvm/examples/BrainF/BrainF.cpp
index 1c7cacba4fff89f56a5701a37457f875cca9c3e4..ac01961735e1376b315ca4f03dacd588887b881e 100644
--- a/llvm/examples/BrainF/BrainF.cpp
+++ b/llvm/examples/BrainF/BrainF.cpp
@@ -37,7 +37,6 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/Casting.h"
 #include <cstdlib>
 #include <iostream>
 
diff --git a/llvm/examples/BrainF/BrainFDriver.cpp b/llvm/examples/BrainF/BrainFDriver.cpp
index 64483479692362e0b716583333fef8d9c5d168bb..98fa735e1491f763367022b8116cf1c8d092deb8 100644
--- a/llvm/examples/BrainF/BrainFDriver.cpp
+++ b/llvm/examples/BrainF/BrainFDriver.cpp
@@ -28,7 +28,6 @@
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
 #include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/MCJIT.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -38,13 +37,11 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/Verifier.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 7d6b8c76079749ec852d816b17c88c1a9a56b5f1..d4a10e927760521ea700f87c6453db9941a2fe05 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -3951,9 +3951,28 @@ const unsigned *LLVMGetIndices(LLVMValueRef Inst);
 
 LLVMBuilderRef LLVMCreateBuilderInContext(LLVMContextRef C);
 LLVMBuilderRef LLVMCreateBuilder(void);
+/**
+ * Set the builder position before Instr but after any attached debug records,
+ * or if Instr is null set the position to the end of Block.
+ */
 void LLVMPositionBuilder(LLVMBuilderRef Builder, LLVMBasicBlockRef Block,
                          LLVMValueRef Instr);
+/**
+ * Set the builder position before Instr and any attached debug records,
+ * or if Instr is null set the position to the end of Block.
+ */
+void LLVMPositionBuilderBeforeDbgRecords(LLVMBuilderRef Builder,
+                                         LLVMBasicBlockRef Block,
+                                         LLVMValueRef Inst);
+/**
+ * Set the builder position before Instr but after any attached debug records.
+ */
 void LLVMPositionBuilderBefore(LLVMBuilderRef Builder, LLVMValueRef Instr);
+/**
+ * Set the builder position before Instr and any attached debug records.
+ */
+void LLVMPositionBuilderBeforeInstrAndDbgRecords(LLVMBuilderRef Builder,
+                                                 LLVMValueRef Instr);
 void LLVMPositionBuilderAtEnd(LLVMBuilderRef Builder, LLVMBasicBlockRef Block);
 LLVMBasicBlockRef LLVMGetInsertBlock(LLVMBuilderRef Builder);
 void LLVMClearInsertionPosition(LLVMBuilderRef Builder);
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index 44a301ecc99280f8022714bdda54359c231a593d..78faadb30d9eb5020238501fd2352ac43e706f6c 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -964,6 +964,13 @@ public:
     return Val;
   }
 
+  /// Factory for Positive and Negative One.
+  ///
+  /// \param Negative True iff the number should be negative.
+  static APFloat getOne(const fltSemantics &Sem, bool Negative = false) {
+    return APFloat(Sem, Negative ? -1 : 1);
+  }
+
   /// Factory for Positive and Negative Infinity.
   ///
   /// \param Negative True iff the number should be negative.
diff --git a/llvm/include/llvm/Analysis/CodeMetrics.h b/llvm/include/llvm/Analysis/CodeMetrics.h
index a9431bca11251a6fafac8e858bc4b8aa914e9b2b..d09018daf95488431b292df083dba65786100458 100644
--- a/llvm/include/llvm/Analysis/CodeMetrics.h
+++ b/llvm/include/llvm/Analysis/CodeMetrics.h
@@ -20,12 +20,15 @@
 namespace llvm {
 class AssumptionCache;
 class BasicBlock;
+class Instruction;
 class Loop;
 class Function;
 template <class T> class SmallPtrSetImpl;
 class TargetTransformInfo;
 class Value;
 
+enum struct ConvergenceKind { None, Controlled, ExtendedLoop, Uncontrolled };
+
 /// Utility to calculate the size and a few similar metrics for a set
 /// of basic blocks.
 struct CodeMetrics {
@@ -42,8 +45,8 @@ struct CodeMetrics {
   /// one or more 'noduplicate' instructions.
   bool notDuplicatable = false;
 
-  /// True if this function contains a call to a convergent function.
-  bool convergent = false;
+  /// The kind of convergence specified in this function.
+  ConvergenceKind Convergence = ConvergenceKind::None;
 
   /// True if this function calls alloca (in the C sense).
   bool usesDynamicAlloca = false;
@@ -77,7 +80,7 @@ struct CodeMetrics {
   /// Add information about a block to the current state.
   void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI,
                          const SmallPtrSetImpl<const Value *> &EphValues,
-                         bool PrepareForLTO = false);
+                         bool PrepareForLTO = false, const Loop *L = nullptr);
 
   /// Collect a loop's ephemeral values (those used only by an assume
   /// or similar intrinsics in the loop).
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index 68c98b416ce963eade98a46ebbd9760de62b7ee1..ac8ca207d312b736b3b5905c7370488403f0253f 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -943,6 +943,11 @@ public:
   LazyCallGraph(LazyCallGraph &&G);
   LazyCallGraph &operator=(LazyCallGraph &&RHS);
 
+#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
+  /// Verify that every RefSCC is valid.
+  void verify();
+#endif
+
   bool invalidate(Module &, const PreservedAnalyses &PA,
                   ModuleAnalysisManager::Invalidator &);
 
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 52084630560c55cae10fdde9ab655998122fabc5..4f06a7e889f912cc0efbf97f5789491102a03269 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -649,6 +649,9 @@ int getIntLoopAttribute(const Loop *TheLoop, StringRef Name, int Default = 0);
 std::optional<const MDOperand *> findStringMetadataForLoop(const Loop *TheLoop,
                                                            StringRef Name);
 
+/// Find the convergence heart of the loop.
+CallBase *getLoopConvergenceHeart(const Loop *TheLoop);
+
 /// Look for the loop attribute that requires progress within the loop.
 /// Note: Most consumers probably want "isMustProgress" which checks
 /// the containing function attribute too.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 72f3d9454249637ea78213de6634b8b36b2c6d4b..b273d118c4c76335834e83a79ec04b8478931b75 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -1656,9 +1656,7 @@ private:
     DenseMap<const SCEV *, ConstantRange> &Cache =
         Hint == HINT_RANGE_UNSIGNED ? UnsignedRanges : SignedRanges;
 
-    auto Pair = Cache.try_emplace(S, std::move(CR));
-    if (!Pair.second)
-      Pair.first->second = std::move(CR);
+    auto Pair = Cache.insert_or_assign(S, std::move(CR));
     return Pair.first->second;
   }
 
diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index e12eb7095b908642cce1f5a7c28fee303bae5d41..ffdf8b8c3bc79fa1c7e11c1fc1bd4f6a0034597c 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -49,6 +49,7 @@ TLI_DEFINE_VECFUNC("llvm.sin.f32", "vsinf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("cosf", "vcosf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "vcosf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tanf", "vtanf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "vtanf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("asinf", "vasinf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("acosf", "vacosf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("atanf", "vatanf", FIXED(4), "_ZGV_LLVM_N4v")
@@ -91,6 +92,11 @@ TLI_DEFINE_VECFUNC("llvm.sin.f64", "_simd_sin_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("sinf", "_simd_sin_f4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.sin.f32", "_simd_sin_f4", FIXED(4), "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("tan", "_simd_tan_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "_simd_tan_d2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("tanf", "_simd_tan_f4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "_simd_tan_f4", FIXED(4), "_ZGV_LLVM_N4v")
+
 // Floating-Point Arithmetic and Auxiliary Functions
 TLI_DEFINE_VECFUNC("cbrt", "_simd_cbrt_d2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("cbrtf", "_simd_cbrt_f4", FIXED(4), "_ZGV_LLVM_N4v")
@@ -142,6 +148,18 @@ TLI_DEFINE_VECFUNC("llvm.cos.f64", "_ZGVdN4v_cos", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVbN4v_cosf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVdN8v_cosf", FIXED(8), "_ZGV_LLVM_N8v")
 
+TLI_DEFINE_VECFUNC("tan", "_ZGVbN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("tan", "_ZGVdN4v_tan", FIXED(4), "_ZGV_LLVM_N4v")
+
+TLI_DEFINE_VECFUNC("tanf", "_ZGVbN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("tanf", "_ZGVdN8v_tanf", FIXED(8), "_ZGV_LLVM_N8v")
+
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVbN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVdN4v_tan", FIXED(4), "_ZGV_LLVM_N4v")
+
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVbN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVdN8v_tanf", FIXED(8), "_ZGV_LLVM_N8v")
+
 TLI_DEFINE_VECFUNC("pow", "_ZGVbN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("pow", "_ZGVdN4vv_pow", FIXED(4), "_ZGV_LLVM_N4vv")
 
@@ -303,6 +321,22 @@ TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf4", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf8", FIXED(8), "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf16", FIXED(16), "_ZGV_LLVM_N16v")
 
+TLI_DEFINE_VECFUNC("tan", "__svml_tan2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("tan", "__svml_tan4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("tan", "__svml_tan8", FIXED(8), "_ZGV_LLVM_N8v")
+
+TLI_DEFINE_VECFUNC("tanf", "__svml_tanf4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("tanf", "__svml_tanf8", FIXED(8), "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("tanf", "__svml_tanf16", FIXED(16), "_ZGV_LLVM_N16v")
+
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan2", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan8", FIXED(8), "_ZGV_LLVM_N8v")
+
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf4", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf8", FIXED(8), "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf16", FIXED(16), "_ZGV_LLVM_N16v")
+
 TLI_DEFINE_VECFUNC("pow", "__svml_pow2", FIXED(2), "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("pow", "__svml_pow4", FIXED(4), "_ZGV_LLVM_N4vv")
 TLI_DEFINE_VECFUNC("pow", "__svml_pow8", FIXED(8), "_ZGV_LLVM_N8vv")
@@ -555,6 +589,7 @@ TLI_DEFINE_VECFUNC("sinpi", "_ZGVnN2v_sinpi", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("sqrt", "_ZGVnN2v_sqrt", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("tan", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC("tanh", "_ZGVnN2v_tanh", FIXED(2), "_ZGV_LLVM_N2v")
 
@@ -652,6 +687,7 @@ TLI_DEFINE_VECFUNC("sinpif", "_ZGVnN4v_sinpif", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("sqrtf", "_ZGVnN4v_sqrtf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("tanf", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC("tanhf", "_ZGVnN4v_tanhf", FIXED(4), "_ZGV_LLVM_N4v")
 
@@ -799,6 +835,8 @@ TLI_DEFINE_VECFUNC("sqrtf", "_ZGVsMxv_sqrtf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("tan", "_ZGVsMxv_tan",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tanf", "_ZGVsMxv_tanf", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVsMxv_tan", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVsMxv_tanf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
 TLI_DEFINE_VECFUNC("tanh", "_ZGVsMxv_tanh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tanhf", "_ZGVsMxv_tanhf", SCALABLE(4), MASKED, "_ZGVsMxv")
@@ -1058,6 +1096,11 @@ TLI_DEFINE_VECFUNC("tanf", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tan", "armpl_svtan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("tanf", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_svtan_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+
 TLI_DEFINE_VECFUNC("tanh", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("tanhf", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tanh", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
@@ -1237,6 +1280,13 @@ TLI_DEFINE_VECFUNC("tanf", "amd_vrs4_tanf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("tanf", "amd_vrs8_tanf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("tanf", "amd_vrs16_tanf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
 
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs16_tanf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs8_tanf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs4_tanf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd8_tan", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd4_tan", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd2_tan", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
+
 TLI_DEFINE_VECFUNC("asin", "amd_vrd8_asin", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
 TLI_DEFINE_VECFUNC("asinf", "amd_vrs4_asinf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("asinf", "amd_vrs8_asinf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v")
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index b2dcdfad0a04b4c64b725280d63a24aec7d069ee..e687254f6c4c70e690ffe5d50e05bc826074905e 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -337,7 +337,6 @@ namespace llvm {
 
     // Top-Level Entities
     bool parseTopLevelEntities();
-    bool finalizeDebugInfoFormat(Module *M);
     void dropUnknownMetadataReferences();
     bool validateEndOfModule(bool UpgradeDebugInfo);
     bool validateEndOfIndex();
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 69821c22dcd6191d76c14c79b779796c5e991434..db6780b70ca5aac3f66bf04a6df12bac88933f2e 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -147,6 +147,7 @@ enum Kind {
   kw_aarch64_vector_pcs,
   kw_aarch64_sve_vector_pcs,
   kw_aarch64_sme_preservemost_from_x0,
+  kw_aarch64_sme_preservemost_from_x1,
   kw_aarch64_sme_preservemost_from_x2,
   kw_msp430_intrcc,
   kw_avr_intrcc,
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index adcf24eb83b031d9e531536da18727434324fed3..d55947fc5103ac592ecf3d2f7ed529b22c9bfc1f 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -51,7 +51,7 @@
 #endif
 
 #ifndef HANDLE_DW_OP
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)
 #endif
 
 #ifndef HANDLE_DW_OP_LLVM_USEROP
@@ -694,197 +694,197 @@ HANDLE_DW_FORM(0x1f21, GNU_strp_alt, 0, GNU)
 HANDLE_DW_FORM(0x2001, LLVM_addrx_offset, 0, LLVM)
 
 // DWARF Expression operators.
-HANDLE_DW_OP(0x03, addr, 2, DWARF)
-HANDLE_DW_OP(0x06, deref, 2, DWARF)
-HANDLE_DW_OP(0x08, const1u, 2, DWARF)
-HANDLE_DW_OP(0x09, const1s, 2, DWARF)
-HANDLE_DW_OP(0x0a, const2u, 2, DWARF)
-HANDLE_DW_OP(0x0b, const2s, 2, DWARF)
-HANDLE_DW_OP(0x0c, const4u, 2, DWARF)
-HANDLE_DW_OP(0x0d, const4s, 2, DWARF)
-HANDLE_DW_OP(0x0e, const8u, 2, DWARF)
-HANDLE_DW_OP(0x0f, const8s, 2, DWARF)
-HANDLE_DW_OP(0x10, constu, 2, DWARF)
-HANDLE_DW_OP(0x11, consts, 2, DWARF)
-HANDLE_DW_OP(0x12, dup, 2, DWARF)
-HANDLE_DW_OP(0x13, drop, 2, DWARF)
-HANDLE_DW_OP(0x14, over, 2, DWARF)
-HANDLE_DW_OP(0x15, pick, 2, DWARF)
-HANDLE_DW_OP(0x16, swap, 2, DWARF)
-HANDLE_DW_OP(0x17, rot, 2, DWARF)
-HANDLE_DW_OP(0x18, xderef, 2, DWARF)
-HANDLE_DW_OP(0x19, abs, 2, DWARF)
-HANDLE_DW_OP(0x1a, and, 2, DWARF)
-HANDLE_DW_OP(0x1b, div, 2, DWARF)
-HANDLE_DW_OP(0x1c, minus, 2, DWARF)
-HANDLE_DW_OP(0x1d, mod, 2, DWARF)
-HANDLE_DW_OP(0x1e, mul, 2, DWARF)
-HANDLE_DW_OP(0x1f, neg, 2, DWARF)
-HANDLE_DW_OP(0x20, not, 2, DWARF)
-HANDLE_DW_OP(0x21, or, 2, DWARF)
-HANDLE_DW_OP(0x22, plus, 2, DWARF)
-HANDLE_DW_OP(0x23, plus_uconst, 2, DWARF)
-HANDLE_DW_OP(0x24, shl, 2, DWARF)
-HANDLE_DW_OP(0x25, shr, 2, DWARF)
-HANDLE_DW_OP(0x26, shra, 2, DWARF)
-HANDLE_DW_OP(0x27, xor, 2, DWARF)
-HANDLE_DW_OP(0x28, bra, 2, DWARF)
-HANDLE_DW_OP(0x29, eq, 2, DWARF)
-HANDLE_DW_OP(0x2a, ge, 2, DWARF)
-HANDLE_DW_OP(0x2b, gt, 2, DWARF)
-HANDLE_DW_OP(0x2c, le, 2, DWARF)
-HANDLE_DW_OP(0x2d, lt, 2, DWARF)
-HANDLE_DW_OP(0x2e, ne, 2, DWARF)
-HANDLE_DW_OP(0x2f, skip, 2, DWARF)
-HANDLE_DW_OP(0x30, lit0, 2, DWARF)
-HANDLE_DW_OP(0x31, lit1, 2, DWARF)
-HANDLE_DW_OP(0x32, lit2, 2, DWARF)
-HANDLE_DW_OP(0x33, lit3, 2, DWARF)
-HANDLE_DW_OP(0x34, lit4, 2, DWARF)
-HANDLE_DW_OP(0x35, lit5, 2, DWARF)
-HANDLE_DW_OP(0x36, lit6, 2, DWARF)
-HANDLE_DW_OP(0x37, lit7, 2, DWARF)
-HANDLE_DW_OP(0x38, lit8, 2, DWARF)
-HANDLE_DW_OP(0x39, lit9, 2, DWARF)
-HANDLE_DW_OP(0x3a, lit10, 2, DWARF)
-HANDLE_DW_OP(0x3b, lit11, 2, DWARF)
-HANDLE_DW_OP(0x3c, lit12, 2, DWARF)
-HANDLE_DW_OP(0x3d, lit13, 2, DWARF)
-HANDLE_DW_OP(0x3e, lit14, 2, DWARF)
-HANDLE_DW_OP(0x3f, lit15, 2, DWARF)
-HANDLE_DW_OP(0x40, lit16, 2, DWARF)
-HANDLE_DW_OP(0x41, lit17, 2, DWARF)
-HANDLE_DW_OP(0x42, lit18, 2, DWARF)
-HANDLE_DW_OP(0x43, lit19, 2, DWARF)
-HANDLE_DW_OP(0x44, lit20, 2, DWARF)
-HANDLE_DW_OP(0x45, lit21, 2, DWARF)
-HANDLE_DW_OP(0x46, lit22, 2, DWARF)
-HANDLE_DW_OP(0x47, lit23, 2, DWARF)
-HANDLE_DW_OP(0x48, lit24, 2, DWARF)
-HANDLE_DW_OP(0x49, lit25, 2, DWARF)
-HANDLE_DW_OP(0x4a, lit26, 2, DWARF)
-HANDLE_DW_OP(0x4b, lit27, 2, DWARF)
-HANDLE_DW_OP(0x4c, lit28, 2, DWARF)
-HANDLE_DW_OP(0x4d, lit29, 2, DWARF)
-HANDLE_DW_OP(0x4e, lit30, 2, DWARF)
-HANDLE_DW_OP(0x4f, lit31, 2, DWARF)
-HANDLE_DW_OP(0x50, reg0, 2, DWARF)
-HANDLE_DW_OP(0x51, reg1, 2, DWARF)
-HANDLE_DW_OP(0x52, reg2, 2, DWARF)
-HANDLE_DW_OP(0x53, reg3, 2, DWARF)
-HANDLE_DW_OP(0x54, reg4, 2, DWARF)
-HANDLE_DW_OP(0x55, reg5, 2, DWARF)
-HANDLE_DW_OP(0x56, reg6, 2, DWARF)
-HANDLE_DW_OP(0x57, reg7, 2, DWARF)
-HANDLE_DW_OP(0x58, reg8, 2, DWARF)
-HANDLE_DW_OP(0x59, reg9, 2, DWARF)
-HANDLE_DW_OP(0x5a, reg10, 2, DWARF)
-HANDLE_DW_OP(0x5b, reg11, 2, DWARF)
-HANDLE_DW_OP(0x5c, reg12, 2, DWARF)
-HANDLE_DW_OP(0x5d, reg13, 2, DWARF)
-HANDLE_DW_OP(0x5e, reg14, 2, DWARF)
-HANDLE_DW_OP(0x5f, reg15, 2, DWARF)
-HANDLE_DW_OP(0x60, reg16, 2, DWARF)
-HANDLE_DW_OP(0x61, reg17, 2, DWARF)
-HANDLE_DW_OP(0x62, reg18, 2, DWARF)
-HANDLE_DW_OP(0x63, reg19, 2, DWARF)
-HANDLE_DW_OP(0x64, reg20, 2, DWARF)
-HANDLE_DW_OP(0x65, reg21, 2, DWARF)
-HANDLE_DW_OP(0x66, reg22, 2, DWARF)
-HANDLE_DW_OP(0x67, reg23, 2, DWARF)
-HANDLE_DW_OP(0x68, reg24, 2, DWARF)
-HANDLE_DW_OP(0x69, reg25, 2, DWARF)
-HANDLE_DW_OP(0x6a, reg26, 2, DWARF)
-HANDLE_DW_OP(0x6b, reg27, 2, DWARF)
-HANDLE_DW_OP(0x6c, reg28, 2, DWARF)
-HANDLE_DW_OP(0x6d, reg29, 2, DWARF)
-HANDLE_DW_OP(0x6e, reg30, 2, DWARF)
-HANDLE_DW_OP(0x6f, reg31, 2, DWARF)
-HANDLE_DW_OP(0x70, breg0, 2, DWARF)
-HANDLE_DW_OP(0x71, breg1, 2, DWARF)
-HANDLE_DW_OP(0x72, breg2, 2, DWARF)
-HANDLE_DW_OP(0x73, breg3, 2, DWARF)
-HANDLE_DW_OP(0x74, breg4, 2, DWARF)
-HANDLE_DW_OP(0x75, breg5, 2, DWARF)
-HANDLE_DW_OP(0x76, breg6, 2, DWARF)
-HANDLE_DW_OP(0x77, breg7, 2, DWARF)
-HANDLE_DW_OP(0x78, breg8, 2, DWARF)
-HANDLE_DW_OP(0x79, breg9, 2, DWARF)
-HANDLE_DW_OP(0x7a, breg10, 2, DWARF)
-HANDLE_DW_OP(0x7b, breg11, 2, DWARF)
-HANDLE_DW_OP(0x7c, breg12, 2, DWARF)
-HANDLE_DW_OP(0x7d, breg13, 2, DWARF)
-HANDLE_DW_OP(0x7e, breg14, 2, DWARF)
-HANDLE_DW_OP(0x7f, breg15, 2, DWARF)
-HANDLE_DW_OP(0x80, breg16, 2, DWARF)
-HANDLE_DW_OP(0x81, breg17, 2, DWARF)
-HANDLE_DW_OP(0x82, breg18, 2, DWARF)
-HANDLE_DW_OP(0x83, breg19, 2, DWARF)
-HANDLE_DW_OP(0x84, breg20, 2, DWARF)
-HANDLE_DW_OP(0x85, breg21, 2, DWARF)
-HANDLE_DW_OP(0x86, breg22, 2, DWARF)
-HANDLE_DW_OP(0x87, breg23, 2, DWARF)
-HANDLE_DW_OP(0x88, breg24, 2, DWARF)
-HANDLE_DW_OP(0x89, breg25, 2, DWARF)
-HANDLE_DW_OP(0x8a, breg26, 2, DWARF)
-HANDLE_DW_OP(0x8b, breg27, 2, DWARF)
-HANDLE_DW_OP(0x8c, breg28, 2, DWARF)
-HANDLE_DW_OP(0x8d, breg29, 2, DWARF)
-HANDLE_DW_OP(0x8e, breg30, 2, DWARF)
-HANDLE_DW_OP(0x8f, breg31, 2, DWARF)
-HANDLE_DW_OP(0x90, regx, 2, DWARF)
-HANDLE_DW_OP(0x91, fbreg, 2, DWARF)
-HANDLE_DW_OP(0x92, bregx, 2, DWARF)
-HANDLE_DW_OP(0x93, piece, 2, DWARF)
-HANDLE_DW_OP(0x94, deref_size, 2, DWARF)
-HANDLE_DW_OP(0x95, xderef_size, 2, DWARF)
-HANDLE_DW_OP(0x96, nop, 2, DWARF)
+HANDLE_DW_OP(0x03, addr, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x06, deref, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x08, const1u, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x09, const1s, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0a, const2u, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0b, const2s, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0c, const4u, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0d, const4s, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0e, const8u, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x0f, const8s, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x10, constu, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x11, consts, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x12, dup, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x13, drop, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x14, over, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x15, pick, 1, -1, 2, DWARF)
+HANDLE_DW_OP(0x16, swap, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x17, rot, 0, 3, 2, DWARF)
+HANDLE_DW_OP(0x18, xderef, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x19, abs, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x1a, and, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x1b, div, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x1c, minus, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x1d, mod, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x1e, mul, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x1f, neg, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x20, not, 0, 1, 2, DWARF)
+HANDLE_DW_OP(0x21, or, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x22, plus, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x23, plus_uconst, 1, 1, 2, DWARF)
+HANDLE_DW_OP(0x24, shl, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x25, shr, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x26, shra, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x27, xor, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x28, bra, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x29, eq, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2a, ge, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2b, gt, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2c, le, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2d, lt, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2e, ne, 0, 2, 2, DWARF)
+HANDLE_DW_OP(0x2f, skip, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x30, lit0, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x31, lit1, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x32, lit2, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x33, lit3, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x34, lit4, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x35, lit5, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x36, lit6, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x37, lit7, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x38, lit8, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x39, lit9, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3a, lit10, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3b, lit11, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3c, lit12, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3d, lit13, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3e, lit14, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x3f, lit15, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x40, lit16, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x41, lit17, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x42, lit18, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x43, lit19, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x44, lit20, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x45, lit21, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x46, lit22, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x47, lit23, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x48, lit24, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x49, lit25, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4a, lit26, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4b, lit27, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4c, lit28, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4d, lit29, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4e, lit30, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x4f, lit31, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x50, reg0, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x51, reg1, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x52, reg2, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x53, reg3, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x54, reg4, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x55, reg5, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x56, reg6, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x57, reg7, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x58, reg8, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x59, reg9, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5a, reg10, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5b, reg11, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5c, reg12, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5d, reg13, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5e, reg14, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x5f, reg15, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x60, reg16, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x61, reg17, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x62, reg18, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x63, reg19, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x64, reg20, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x65, reg21, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x66, reg22, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x67, reg23, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x68, reg24, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x69, reg25, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6a, reg26, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6b, reg27, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6c, reg28, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6d, reg29, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6e, reg30, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x6f, reg31, 0, 0, 2, DWARF)
+HANDLE_DW_OP(0x70, breg0, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x71, breg1, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x72, breg2, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x73, breg3, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x74, breg4, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x75, breg5, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x76, breg6, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x77, breg7, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x78, breg8, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x79, breg9, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7a, breg10, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7b, breg11, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7c, breg12, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7d, breg13, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7e, breg14, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x7f, breg15, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x80, breg16, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x81, breg17, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x82, breg18, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x83, breg19, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x84, breg20, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x85, breg21, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x86, breg22, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x87, breg23, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x88, breg24, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x89, breg25, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8a, breg26, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8b, breg27, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8c, breg28, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8d, breg29, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8e, breg30, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x8f, breg31, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x90, regx, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x91, fbreg, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x92, bregx, 2, 0, 2, DWARF)
+HANDLE_DW_OP(0x93, piece, 1, 0, 2, DWARF)
+HANDLE_DW_OP(0x94, deref_size, 1, 1, 2, DWARF)
+HANDLE_DW_OP(0x95, xderef_size, 1, 1, 2, DWARF)
+HANDLE_DW_OP(0x96, nop, 0, 0, 2, DWARF)
 // New in DWARF v3:
-HANDLE_DW_OP(0x97, push_object_address, 3, DWARF)
-HANDLE_DW_OP(0x98, call2, 3, DWARF)
-HANDLE_DW_OP(0x99, call4, 3, DWARF)
-HANDLE_DW_OP(0x9a, call_ref, 3, DWARF)
-HANDLE_DW_OP(0x9b, form_tls_address, 3, DWARF)
-HANDLE_DW_OP(0x9c, call_frame_cfa, 3, DWARF)
-HANDLE_DW_OP(0x9d, bit_piece, 3, DWARF)
+HANDLE_DW_OP(0x97, push_object_address, 0, 0, 3, DWARF)
+HANDLE_DW_OP(0x98, call2, 1, 0, 3, DWARF)
+HANDLE_DW_OP(0x99, call4, 1, 0, 3, DWARF)
+HANDLE_DW_OP(0x9a, call_ref, 1, 1, 3, DWARF)
+HANDLE_DW_OP(0x9b, form_tls_address, 0, 1, 3, DWARF)
+HANDLE_DW_OP(0x9c, call_frame_cfa, 0, 0, 3, DWARF)
+HANDLE_DW_OP(0x9d, bit_piece, 2, 0, 3, DWARF)
 // New in DWARF v4:
-HANDLE_DW_OP(0x9e, implicit_value, 4, DWARF)
-HANDLE_DW_OP(0x9f, stack_value, 4, DWARF)
+HANDLE_DW_OP(0x9e, implicit_value, 2, 0, 4, DWARF)
+HANDLE_DW_OP(0x9f, stack_value, 0, 1, 4, DWARF)
 // New in DWARF v5:
-HANDLE_DW_OP(0xa0, implicit_pointer, 5, DWARF)
-HANDLE_DW_OP(0xa1, addrx, 5, DWARF)
-HANDLE_DW_OP(0xa2, constx, 5, DWARF)
-HANDLE_DW_OP(0xa3, entry_value, 5, DWARF)
-HANDLE_DW_OP(0xa4, const_type, 5, DWARF)
-HANDLE_DW_OP(0xa5, regval_type, 5, DWARF)
-HANDLE_DW_OP(0xa6, deref_type, 5, DWARF)
-HANDLE_DW_OP(0xa7, xderef_type, 5, DWARF)
-HANDLE_DW_OP(0xa8, convert, 5, DWARF)
-HANDLE_DW_OP(0xa9, reinterpret, 5, DWARF)
+HANDLE_DW_OP(0xa0, implicit_pointer, 2, 0, 5, DWARF)
+HANDLE_DW_OP(0xa1, addrx, 1, 0, 5, DWARF)
+HANDLE_DW_OP(0xa2, constx, 1, 0, 5, DWARF)
+HANDLE_DW_OP(0xa3, entry_value, 2, 0, 5, DWARF)
+HANDLE_DW_OP(0xa4, const_type, 3, 0, 5, DWARF)
+HANDLE_DW_OP(0xa5, regval_type, 2, 0, 5, DWARF)
+HANDLE_DW_OP(0xa6, deref_type, 2, 1, 5, DWARF)
+HANDLE_DW_OP(0xa7, xderef_type, 2, 2, 5, DWARF)
+HANDLE_DW_OP(0xa8, convert, 2, 1, 5, DWARF)
+HANDLE_DW_OP(0xa9, reinterpret, 1, 1, 5, DWARF)
 // Vendor extensions:
 // Extensions for GNU-style thread-local storage.
-HANDLE_DW_OP(0xe0, GNU_push_tls_address, 0, GNU)
+HANDLE_DW_OP(0xe0, GNU_push_tls_address, 0, 0, 0, GNU)
 // Conflicting:
-// HANDLE_DW_OP(0xe0, HP_unknown, 0, HP)
-HANDLE_DW_OP(0xe1, HP_is_value, 0, HP)
-HANDLE_DW_OP(0xe2, HP_fltconst4, 0, HP)
-HANDLE_DW_OP(0xe3, HP_fltconst8, 0, HP)
-HANDLE_DW_OP(0xe4, HP_mod_range, 0, HP)
-HANDLE_DW_OP(0xe5, HP_unmod_range, 0, HP)
-HANDLE_DW_OP(0xe6, HP_tls, 0, HP)
-HANDLE_DW_OP(0xe8, INTEL_bit_piece, 0, INTEL)
+// HANDLE_DW_OP(0xe0, HP_unknown, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe1, HP_is_value, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe2, HP_fltconst4, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe3, HP_fltconst8, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe4, HP_mod_range, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe5, HP_unmod_range, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe6, HP_tls, -1, -1, 0, HP)
+HANDLE_DW_OP(0xe8, INTEL_bit_piece, -1, -1, 0, INTEL)
 
 // Extensions for WebAssembly.
-HANDLE_DW_OP(0xed, WASM_location, 0, WASM)
-HANDLE_DW_OP(0xee, WASM_location_int, 0, WASM)
+HANDLE_DW_OP(0xed, WASM_location, -1, -1, 0, WASM)
+HANDLE_DW_OP(0xee, WASM_location_int, -1, -1, 0, WASM)
 // Historic and not implemented in LLVM.
-HANDLE_DW_OP(0xf0, APPLE_uninit, 0, APPLE)
+HANDLE_DW_OP(0xf0, APPLE_uninit, -1, -1, 0, APPLE)
 // The GNU entry value extension.
-HANDLE_DW_OP(0xf3, GNU_entry_value, 0, GNU)
-HANDLE_DW_OP(0xf8, PGI_omp_thread_num, 0, PGI)
+HANDLE_DW_OP(0xf3, GNU_entry_value, 2, 0, 0, GNU)
+HANDLE_DW_OP(0xf8, PGI_omp_thread_num, -1, -1, 0, PGI)
 // Extensions for Fission proposal.
-HANDLE_DW_OP(0xfb, GNU_addr_index, 0, GNU)
-HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU)
+HANDLE_DW_OP(0xfb, GNU_addr_index, 1, 0, 0, GNU)
+HANDLE_DW_OP(0xfc, GNU_const_index, 1, 0, 0, GNU)
 
 // DW_OP_LLVM_user has two operands:
 //   (1) An unsigned LEB128 "LLVM Vendor Extension Opcode".
@@ -893,7 +893,7 @@ HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU)
 // DW_OP_LLVM_user acts as an extension multiplexer, opening up the encoding
 // space to accommodate an infinite number of extensions. This better reflects
 // the de-facto permanent allocation of extensions.
-HANDLE_DW_OP(0xe9, LLVM_user, 0, LLVM)
+HANDLE_DW_OP(0xe9, LLVM_user, -1, -1, 0, LLVM)
 // "LLVM Vendor Extension" operations under the DW_OP_LLVM_user encoding
 // scheme. This list is authoritative and exhaustive. Once an operation is
 // registered here it cannot be removed nor have its encoding changed. The
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index 74c4d6ff3a7167cfb1f780f7d1d018e34e289b00..4657ad30eb1be47c5a6b185cedc04fe466d23a8c 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -134,16 +134,19 @@ enum Form : uint16_t {
 };
 
 enum LocationAtom {
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR) DW_OP_##NAME = ID,
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
+  DW_OP_##NAME = ID,
 #include "llvm/BinaryFormat/Dwarf.def"
   DW_OP_lo_user = 0xe0,
   DW_OP_hi_user = 0xff,
-  DW_OP_LLVM_fragment = 0x1000,         ///< Only used in LLVM metadata.
-  DW_OP_LLVM_convert = 0x1001,          ///< Only used in LLVM metadata.
-  DW_OP_LLVM_tag_offset = 0x1002,       ///< Only used in LLVM metadata.
-  DW_OP_LLVM_entry_value = 0x1003,      ///< Only used in LLVM metadata.
-  DW_OP_LLVM_implicit_pointer = 0x1004, ///< Only used in LLVM metadata.
-  DW_OP_LLVM_arg = 0x1005,              ///< Only used in LLVM metadata.
+  DW_OP_LLVM_fragment = 0x1000,          ///< Only used in LLVM metadata.
+  DW_OP_LLVM_convert = 0x1001,           ///< Only used in LLVM metadata.
+  DW_OP_LLVM_tag_offset = 0x1002,        ///< Only used in LLVM metadata.
+  DW_OP_LLVM_entry_value = 0x1003,       ///< Only used in LLVM metadata.
+  DW_OP_LLVM_implicit_pointer = 0x1004,  ///< Only used in LLVM metadata.
+  DW_OP_LLVM_arg = 0x1005,               ///< Only used in LLVM metadata.
+  DW_OP_LLVM_extract_bits_sext = 0x1006, ///< Only used in LLVM metadata.
+  DW_OP_LLVM_extract_bits_zext = 0x1007, ///< Only used in LLVM metadata.
 };
 
 enum LlvmUserLocationAtom {
@@ -1047,6 +1050,14 @@ unsigned AttributeEncodingVendor(TypeKind E);
 unsigned LanguageVendor(SourceLanguage L);
 /// @}
 
+/// The number of operands for the given LocationAtom.
+std::optional<unsigned> OperationOperands(LocationAtom O);
+
+/// The arity of the given LocationAtom. This is the number of elements on the
+/// stack this operation operates on. Returns -1 if the arity is variable (e.g.
+/// depending on the argument) or unknown.
+std::optional<unsigned> OperationArity(LocationAtom O);
+
 std::optional<unsigned> LanguageLowerBound(SourceLanguage L);
 
 /// The size of a reference determined by the DWARF 32/64-bit format.
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 7364d619bc1b134e5f4306b1f3a2eed29d4fce37..dfba180149916815a8fead60847dfc096dda1ecb 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -795,7 +795,7 @@ enum : unsigned {
   EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC = 0x052,
   EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC = 0x053,
   EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC   = 0x054,
-  EF_AMDGPU_MACH_AMDGCN_RESERVED_0X55   = 0x055,
+  EF_AMDGPU_MACH_AMDGCN_GFX1152         = 0x055,
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X56   = 0x056,
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X57   = 0x057,
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X58   = 0x058,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index ef4e0fd29768f9d8a7c16274ce40fccc12bde5ef..9f8d3ded9b3c1a86eab15ecc9feb001debef81b3 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1977,6 +1977,9 @@ public:
     case Intrinsic::cos:
       ISD = ISD::FCOS;
       break;
+    case Intrinsic::tan:
+      ISD = ISD::FTAN;
+      break;
     case Intrinsic::exp:
       ISD = ISD::FEXP;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 12e5b31e5817acac20bb99c8750a06b79bb1c850..43659564d5ace72589d238fdd634dc575c946588 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -871,6 +871,14 @@ public:
   bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI,
                                              BuildFnTy &MatchInfo);
 
+  bool matchAddOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
+  bool matchMulOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
+  bool matchSubOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
+  bool matchShlOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
 private:
   /// Checks for legality of an indexed variant of \p LdSt.
   bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 2b3efc3b609f0fb8dfd8652a93177d327c9e422f..22737256377133c813b5135e84340e286d745fc0 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -14,10 +14,12 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
 #define LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
 
-#include "llvm/IR/Instructions.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/Support/Casting.h"
 
 namespace llvm {
@@ -856,6 +858,43 @@ public:
   };
 };
 
+/// Represents a vscale.
+class GVScale : public GenericMachineInstr {
+public:
+  APInt getSrc() const { return getOperand(1).getCImm()->getValue(); }
+
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_VSCALE;
+  };
+};
+
+/// Represents an integer subtraction.
+class GSub : public GIntBinOp {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_SUB;
+  };
+};
+
+/// Represents an integer multiplication.
+class GMul : public GIntBinOp {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_MUL;
+  };
+};
+
+/// Represents a shift left.
+class GShl : public GenericMachineInstr {
+public:
+  Register getSrcReg() const { return getOperand(1).getReg(); }
+  Register getShiftReg() const { return getOperand(2).getReg(); }
+
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_SHL;
+  };
+};
+
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 0f87e062e2da60bbc3a042b6f434a6457d1e321d..c8c86ed5eef297bc0c0e0a95231a8aa591db90d6 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -415,6 +415,7 @@ enum NodeType {
   STRICT_FLDEXP,
   STRICT_FSIN,
   STRICT_FCOS,
+  STRICT_FTAN,
   STRICT_FEXP,
   STRICT_FEXP2,
   STRICT_FLOG,
@@ -934,6 +935,7 @@ enum NodeType {
   FCBRT,
   FSIN,
   FCOS,
+  FTAN,
   FPOW,
   FPOWI,
   /// FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 9f8e846cac45b9d33165848da6db5f4b1c498a4e..6e7292abeddbbdf522bf2cbefe1537def01614ad 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -18,7 +18,6 @@
 #define LLVM_CODEGEN_MACHINEFUNCTION_H
 
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/SmallVector.h"
@@ -34,6 +33,7 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Recycler.h"
 #include "llvm/Target/TargetOptions.h"
+#include <bitset>
 #include <cassert>
 #include <cstdint>
 #include <memory>
@@ -217,22 +217,21 @@ public:
   }
 
   MachineFunctionProperties &reset(const MachineFunctionProperties &MFP) {
-    Properties.reset(MFP.Properties);
+    Properties &= ~MFP.Properties;
     return *this;
   }
 
   // Returns true if all properties set in V (i.e. required by a pass) are set
   // in this.
   bool verifyRequiredProperties(const MachineFunctionProperties &V) const {
-    return !V.Properties.test(Properties);
+    return (Properties | ~V.Properties).all();
   }
 
   /// Print the MachineFunctionProperties in human-readable form.
   void print(raw_ostream &OS) const;
 
 private:
-  BitVector Properties =
-      BitVector(static_cast<unsigned>(Property::LastProperty)+1);
+  std::bitset<static_cast<unsigned>(Property::LastProperty) + 1> Properties;
 };
 
 struct SEHHandler {
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 852b1a0f416139e508876630671ed427d0236e57..7d15664fbe7548db367a273a627a64c807d965a5 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -61,7 +61,7 @@ struct MachinePassModel
 #ifndef NDEBUG
     if constexpr (is_detected<has_get_required_properties_t, PassT>::value) {
       auto &MFProps = IR.getProperties();
-      auto RequiredProperties = PassT::getRequiredProperties();
+      auto RequiredProperties = this->Pass.getRequiredProperties();
       if (!MFProps.verifyRequiredProperties(RequiredProperties)) {
         errs() << "MachineFunctionProperties required by " << PassT::name()
                << " pass are not met by function " << IR.getName() << ".\n"
@@ -78,9 +78,9 @@ struct MachinePassModel
     auto PA = this->Pass.run(IR, AM);
 
     if constexpr (is_detected<has_get_set_properties_t, PassT>::value)
-      IR.getProperties().set(PassT::getSetProperties());
+      IR.getProperties().set(this->Pass.getSetProperties());
     if constexpr (is_detected<has_get_cleared_properties_t, PassT>::value)
-      IR.getProperties().reset(PassT::getClearedProperties());
+      IR.getProperties().reset(this->Pass.getClearedProperties());
     return PA;
   }
 
diff --git a/llvm/include/llvm/CodeGen/RegAllocFast.h b/llvm/include/llvm/CodeGen/RegAllocFast.h
new file mode 100644
index 0000000000000000000000000000000000000000..c50deccabd9955ea63b8196cc552f0ceef74976b
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/RegAllocFast.h
@@ -0,0 +1,57 @@
+//==- RegAllocFast.h ----------- fast register allocator  ----------*-C++-*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGALLOCFAST_H
+#define LLVM_CODEGEN_REGALLOCFAST_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/RegAllocCommon.h"
+
+namespace llvm {
+
+struct RegAllocFastPassOptions {
+  RegClassFilterFunc Filter = allocateAllRegClasses;
+  StringRef FilterName = "all";
+  bool ClearVRegs = true;
+};
+
+class RegAllocFastPass : public PassInfoMixin<RegAllocFastPass> {
+  RegAllocFastPassOptions Opts;
+
+public:
+  RegAllocFastPass(RegAllocFastPassOptions Opts = RegAllocFastPassOptions())
+      : Opts(Opts) {}
+
+  MachineFunctionProperties getRequiredProperties() {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+
+  MachineFunctionProperties getSetProperties() {
+    if (Opts.ClearVRegs) {
+      return MachineFunctionProperties().set(
+          MachineFunctionProperties::Property::NoVRegs);
+    }
+
+    return MachineFunctionProperties();
+  }
+
+  MachineFunctionProperties getClearedProperties() {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+
+  PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &);
+
+  void printPipeline(raw_ostream &OS,
+                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_REGALLOCFAST_H
diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 7970441c83715bca684d846d94ec8d05d2347617..071a27a79506422d42317a77140e0d27d1cad609 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -330,9 +330,7 @@ template <typename... Preds> struct And {
 template <typename Pred, typename... Preds>
 struct And<Pred, Preds...> : And<Preds...> {
   Pred P;
-  And(Pred &&p, Preds &&...preds)
-      : And<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {
-  }
+  And(const Pred &p, const Preds &...preds) : And<Preds...>(preds...), P(p) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
@@ -349,8 +347,7 @@ template <typename... Preds> struct Or {
 template <typename Pred, typename... Preds>
 struct Or<Pred, Preds...> : Or<Preds...> {
   Pred P;
-  Or(Pred &&p, Preds &&...preds)
-      : Or<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {}
+  Or(const Pred &p, const Preds &...preds) : Or<Preds...>(preds...), P(p) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
@@ -376,16 +373,16 @@ template <typename Pred> inline Not<Pred> m_Unless(const Pred &P) {
   return Not{P};
 }
 
-template <typename... Preds> And<Preds...> m_AllOf(Preds &&...preds) {
-  return And<Preds...>(std::forward<Preds>(preds)...);
+template <typename... Preds> And<Preds...> m_AllOf(const Preds &...preds) {
+  return And<Preds...>(preds...);
 }
 
-template <typename... Preds> Or<Preds...> m_AnyOf(Preds &&...preds) {
-  return Or<Preds...>(std::forward<Preds>(preds)...);
+template <typename... Preds> Or<Preds...> m_AnyOf(const Preds &...preds) {
+  return Or<Preds...>(preds...);
 }
 
-template <typename... Preds> auto m_NoneOf(Preds &&...preds) {
-  return m_Unless(m_AnyOf(std::forward<Preds>(preds)...));
+template <typename... Preds> auto m_NoneOf(const Preds &...preds) {
+  return m_Unless(m_AnyOf(preds...));
 }
 
 // === Generic node matching ===
@@ -402,10 +399,8 @@ struct Operands_match<OpIdx, OpndPred, OpndPreds...>
     : Operands_match<OpIdx + 1, OpndPreds...> {
   OpndPred P;
 
-  Operands_match(OpndPred &&p, OpndPreds &&...preds)
-      : Operands_match<OpIdx + 1, OpndPreds...>(
-            std::forward<OpndPreds>(preds)...),
-        P(std::forward<OpndPred>(p)) {}
+  Operands_match(const OpndPred &p, const OpndPreds &...preds)
+      : Operands_match<OpIdx + 1, OpndPreds...>(preds...), P(p) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
@@ -419,9 +414,8 @@ struct Operands_match<OpIdx, OpndPred, OpndPreds...>
 };
 
 template <typename... OpndPreds>
-auto m_Node(unsigned Opcode, OpndPreds &&...preds) {
-  return m_AllOf(m_Opc(Opcode), Operands_match<0, OpndPreds...>(
-                                    std::forward<OpndPreds>(preds)...));
+auto m_Node(unsigned Opcode, const OpndPreds &...preds) {
+  return m_AllOf(m_Opc(Opcode), Operands_match<0, OpndPreds...>(preds...));
 }
 
 /// Provide number of operands that are not chain or glue, as well as the first
@@ -647,10 +641,9 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op);
 }
 
-template <typename Opnd> inline auto m_SExt(Opnd &&Op) {
-  return m_AnyOf(
-      UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op),
-      m_Node(ISD::SIGN_EXTEND_INREG, std::forward<Opnd>(Op), m_Value()));
+template <typename Opnd> inline auto m_SExt(const Opnd &Op) {
+  return m_AnyOf(UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op),
+                 m_Node(ISD::SIGN_EXTEND_INREG, Op, m_Value()));
 }
 
 template <typename Opnd> inline UnaryOpc_match<Opnd> m_AnyExt(const Opnd &Op) {
@@ -663,30 +656,28 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_Trunc(const Opnd &Op) {
 
 /// Match a zext or identity
 /// Allows to peek through optional extensions
-template <typename Opnd> inline auto m_ZExtOrSelf(Opnd &&Op) {
-  return m_AnyOf(m_ZExt(std::forward<Opnd>(Op)), std::forward<Opnd>(Op));
+template <typename Opnd> inline auto m_ZExtOrSelf(const Opnd &Op) {
+  return m_AnyOf(m_ZExt(Op), Op);
 }
 
 /// Match a sext or identity
 /// Allows to peek through optional extensions
-template <typename Opnd> inline auto m_SExtOrSelf(Opnd &&Op) {
-  return m_AnyOf(m_SExt(std::forward<Opnd>(Op)), std::forward<Opnd>(Op));
+template <typename Opnd> inline auto m_SExtOrSelf(const Opnd &Op) {
+  return m_AnyOf(m_SExt(Op), Op);
 }
 
 /// Match a aext or identity
 /// Allows to peek through optional extensions
 template <typename Opnd>
-inline Or<UnaryOpc_match<Opnd>, Opnd> m_AExtOrSelf(Opnd &&Op) {
-  return Or<UnaryOpc_match<Opnd>, Opnd>(m_AnyExt(std::forward<Opnd>(Op)),
-                                        std::forward<Opnd>(Op));
+inline Or<UnaryOpc_match<Opnd>, Opnd> m_AExtOrSelf(const Opnd &Op) {
+  return Or<UnaryOpc_match<Opnd>, Opnd>(m_AnyExt(Op), Op);
 }
 
 /// Match a trunc or identity
 /// Allows to peek through optional truncations
 template <typename Opnd>
-inline Or<UnaryOpc_match<Opnd>, Opnd> m_TruncOrSelf(Opnd &&Op) {
-  return Or<UnaryOpc_match<Opnd>, Opnd>(m_Trunc(std::forward<Opnd>(Op)),
-                                        std::forward<Opnd>(Op));
+inline Or<UnaryOpc_match<Opnd>, Opnd> m_TruncOrSelf(const Opnd &Op) {
+  return Or<UnaryOpc_match<Opnd>, Opnd>(m_Trunc(Op), Op);
 }
 
 // === Constants ===
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAG.h b/llvm/include/llvm/CodeGen/ScheduleDAG.h
index c5172e8c542b73b33016b8f89edc3276268385a8..53265b286c7c87e566a59acd2aa173f305ad9e9e 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAG.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAG.h
@@ -243,8 +243,10 @@ class TargetRegisterInfo;
   private:
     enum : unsigned { BoundaryID = ~0u };
 
-    SDNode *Node = nullptr;        ///< Representative node.
-    MachineInstr *Instr = nullptr; ///< Alternatively, a MachineInstr.
+    union {
+      SDNode *Node;        ///< Representative node.
+      MachineInstr *Instr; ///< Alternatively, a MachineInstr.
+    };
 
   public:
     SUnit *OrigNode = nullptr; ///< If not this, the node from which this node
@@ -253,6 +255,10 @@ class TargetRegisterInfo;
     const MCSchedClassDesc *SchedClass =
         nullptr; ///< nullptr or resolved SchedClass.
 
+    const TargetRegisterClass *CopyDstRC =
+        nullptr; ///< Is a special copy node if != nullptr.
+    const TargetRegisterClass *CopySrcRC = nullptr;
+
     SmallVector<SDep, 4> Preds;  ///< All sunit predecessors.
     SmallVector<SDep, 4> Succs;  ///< All sunit successors.
 
@@ -269,8 +275,14 @@ class TargetRegisterInfo;
     unsigned NumSuccsLeft = 0;         ///< # of succs not scheduled.
     unsigned WeakPredsLeft = 0;        ///< # of weak preds not scheduled.
     unsigned WeakSuccsLeft = 0;        ///< # of weak succs not scheduled.
-    unsigned short NumRegDefsLeft = 0; ///< # of reg defs with no scheduled use.
-    unsigned short Latency = 0;        ///< Node latency.
+    unsigned TopReadyCycle = 0; ///< Cycle relative to start when node is ready.
+    unsigned BotReadyCycle = 0; ///< Cycle relative to end when node is ready.
+
+  private:
+    unsigned Depth = 0;  ///< Node depth.
+    unsigned Height = 0; ///< Node height.
+
+  public:
     bool isVRegCycle      : 1;         ///< May use and def the same vreg.
     bool isCall           : 1;         ///< Is a function call.
     bool isCallOp         : 1;         ///< Is a function call operand.
@@ -287,52 +299,54 @@ class TargetRegisterInfo;
     bool isCloned         : 1;         ///< True if this node has been cloned.
     bool isUnbuffered     : 1;         ///< Uses an unbuffered resource.
     bool hasReservedResource : 1;      ///< Uses a reserved resource.
-    Sched::Preference SchedulingPref = Sched::None; ///< Scheduling preference.
+    unsigned short NumRegDefsLeft = 0; ///< # of reg defs with no scheduled use.
+    unsigned short Latency = 0;        ///< Node latency.
 
   private:
     bool isDepthCurrent   : 1;         ///< True if Depth is current.
     bool isHeightCurrent  : 1;         ///< True if Height is current.
-    unsigned Depth = 0;                ///< Node depth.
-    unsigned Height = 0;               ///< Node height.
+    bool isNode : 1; ///< True if the representative is an SDNode
+    bool isInst : 1; ///< True if the representative is a MachineInstr
 
   public:
-    unsigned TopReadyCycle = 0; ///< Cycle relative to start when node is ready.
-    unsigned BotReadyCycle = 0; ///< Cycle relative to end when node is ready.
-
-    const TargetRegisterClass *CopyDstRC =
-        nullptr; ///< Is a special copy node if != nullptr.
-    const TargetRegisterClass *CopySrcRC = nullptr;
+    Sched::Preference SchedulingPref : 4; ///< Scheduling preference.
+    static_assert(Sched::Preference::Last <= (1 << 4),
+                  "not enough bits in bitfield");
 
     /// Constructs an SUnit for pre-regalloc scheduling to represent an
     /// SDNode and any nodes flagged to it.
     SUnit(SDNode *node, unsigned nodenum)
-      : Node(node), NodeNum(nodenum), isVRegCycle(false), isCall(false),
-        isCallOp(false), isTwoAddress(false), isCommutable(false),
-        hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false),
-        isPending(false), isAvailable(false), isScheduled(false),
-        isScheduleHigh(false), isScheduleLow(false), isCloned(false),
-        isUnbuffered(false), hasReservedResource(false), isDepthCurrent(false),
-        isHeightCurrent(false) {}
+        : Node(node), NodeNum(nodenum), isVRegCycle(false), isCall(false),
+          isCallOp(false), isTwoAddress(false), isCommutable(false),
+          hasPhysRegUses(false), hasPhysRegDefs(false),
+          hasPhysRegClobbers(false), isPending(false), isAvailable(false),
+          isScheduled(false), isScheduleHigh(false), isScheduleLow(false),
+          isCloned(false), isUnbuffered(false), hasReservedResource(false),
+          isDepthCurrent(false), isHeightCurrent(false), isNode(true),
+          isInst(false), SchedulingPref(Sched::None) {}
 
     /// Constructs an SUnit for post-regalloc scheduling to represent a
     /// MachineInstr.
     SUnit(MachineInstr *instr, unsigned nodenum)
-      : Instr(instr), NodeNum(nodenum), isVRegCycle(false), isCall(false),
-        isCallOp(false), isTwoAddress(false), isCommutable(false),
-        hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false),
-        isPending(false), isAvailable(false), isScheduled(false),
-        isScheduleHigh(false), isScheduleLow(false), isCloned(false),
-        isUnbuffered(false), hasReservedResource(false), isDepthCurrent(false),
-        isHeightCurrent(false) {}
+        : Instr(instr), NodeNum(nodenum), isVRegCycle(false), isCall(false),
+          isCallOp(false), isTwoAddress(false), isCommutable(false),
+          hasPhysRegUses(false), hasPhysRegDefs(false),
+          hasPhysRegClobbers(false), isPending(false), isAvailable(false),
+          isScheduled(false), isScheduleHigh(false), isScheduleLow(false),
+          isCloned(false), isUnbuffered(false), hasReservedResource(false),
+          isDepthCurrent(false), isHeightCurrent(false), isNode(false),
+          isInst(true), SchedulingPref(Sched::None) {}
 
     /// Constructs a placeholder SUnit.
     SUnit()
-      : isVRegCycle(false), isCall(false), isCallOp(false), isTwoAddress(false),
-        isCommutable(false), hasPhysRegUses(false), hasPhysRegDefs(false),
-        hasPhysRegClobbers(false), isPending(false), isAvailable(false),
-        isScheduled(false), isScheduleHigh(false), isScheduleLow(false),
-        isCloned(false), isUnbuffered(false), hasReservedResource(false),
-        isDepthCurrent(false), isHeightCurrent(false) {}
+        : Node(nullptr), isVRegCycle(false), isCall(false), isCallOp(false),
+          isTwoAddress(false), isCommutable(false), hasPhysRegUses(false),
+          hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),
+          isAvailable(false), isScheduled(false), isScheduleHigh(false),
+          isScheduleLow(false), isCloned(false), isUnbuffered(false),
+          hasReservedResource(false), isDepthCurrent(false),
+          isHeightCurrent(false), isNode(false), isInst(false),
+          SchedulingPref(Sched::None) {}
 
     /// Boundary nodes are placeholders for the boundary of the
     /// scheduling region.
@@ -346,32 +360,36 @@ class TargetRegisterInfo;
     /// Assigns the representative SDNode for this SUnit. This may be used
     /// during pre-regalloc scheduling.
     void setNode(SDNode *N) {
-      assert(!Instr && "Setting SDNode of SUnit with MachineInstr!");
+      assert(!isInst && "Setting SDNode of SUnit with MachineInstr!");
       Node = N;
+      isNode = true;
     }
 
     /// Returns the representative SDNode for this SUnit. This may be used
     /// during pre-regalloc scheduling.
     SDNode *getNode() const {
-      assert(!Instr && "Reading SDNode of SUnit with MachineInstr!");
+      assert(!isInst && (isNode || !Instr) &&
+             "Reading SDNode of SUnit without SDNode!");
       return Node;
     }
 
     /// Returns true if this SUnit refers to a machine instruction as
     /// opposed to an SDNode.
-    bool isInstr() const { return Instr; }
+    bool isInstr() const { return isInst && Instr; }
 
     /// Assigns the instruction for the SUnit. This may be used during
     /// post-regalloc scheduling.
     void setInstr(MachineInstr *MI) {
-      assert(!Node && "Setting MachineInstr of SUnit with SDNode!");
+      assert(!isNode && "Setting MachineInstr of SUnit with SDNode!");
       Instr = MI;
+      isInst = true;
     }
 
     /// Returns the representative MachineInstr for this SUnit. This may be used
     /// during post-regalloc scheduling.
     MachineInstr *getInstr() const {
-      assert(!Node && "Reading MachineInstr of SUnit with SDNode!");
+      assert(!isNode && (isInst || !Node) &&
+             "Reading MachineInstr of SUnit without MachineInstr!");
       return Instr;
     }
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 48cb0cdf851cf067b2b5a24f55353c2b0e9e0d7b..7b0e5e7d9504bff19f3bee6775d38f176a8b1402 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1893,7 +1893,8 @@ public:
                            const SDNode *N2);
 
   SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
-                                 ArrayRef<SDValue> Ops);
+                                 ArrayRef<SDValue> Ops,
+                                 SDNodeFlags Flags = SDNodeFlags());
 
   /// Fold floating-point operations when all operands are constants and/or
   /// undefined.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 9a54dbd434d92a463b8f181dc93d3a3205e9490a..2edb039cbbfd2a3f500bf4b25d0bfb282282c392 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -649,7 +649,9 @@ private:
   /// Return a pointer to the specified value type.
   static const EVT *getValueTypeList(EVT VT);
 
-  /// Index in worklist of DAGCombiner, or -1.
+  /// Index in worklist of DAGCombiner, or negative if the node is not in the
+  /// worklist. -1 = not in worklist; -2 = not in worklist, but has already been
+  /// combined at least once.
   int CombinerWorklistIndex = -1;
 
   uint32_t CFIType = 0;
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index d1912b1c4c0f6ebef32993d91661776516675e79..974a5301f6e294fa1508c4decd7d30a1e367ec37 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -96,15 +96,16 @@ class Value;
 
 namespace Sched {
 
-enum Preference {
-  None,        // No preference
-  Source,      // Follow source order.
-  RegPressure, // Scheduling for lowest register pressure.
-  Hybrid,      // Scheduling for both latency and register pressure.
-  ILP,         // Scheduling for ILP in low register pressure mode.
-  VLIW,        // Scheduling for VLIW targets.
-  Fast,        // Fast suboptimal list scheduling
-  Linearize    // Linearize DAG, no scheduling
+enum Preference : uint8_t {
+  None,            // No preference
+  Source,          // Follow source order.
+  RegPressure,     // Scheduling for lowest register pressure.
+  Hybrid,          // Scheduling for both latency and register pressure.
+  ILP,             // Scheduling for ILP in low register pressure mode.
+  VLIW,            // Scheduling for VLIW targets.
+  Fast,            // Fast suboptimal list scheduling
+  Linearize,       // Linearize DAG, no scheduling
+  Last = Linearize // Marker for the last Sched::Preference
 };
 
 } // end namespace Sched
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index bac923aba02afdf5ce32c916b7e4c360019b1782..6ce6affe38e059aaa7026b855af6c0845cce813b 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1438,7 +1438,7 @@ class ExecutionSession {
 
 public:
   /// For reporting errors.
-  using ErrorReporter = std::function<void(Error)>;
+  using ErrorReporter = unique_function<void(Error)>;
 
   /// Send a result to the remote.
   using SendResultFunction = unique_function<void(shared::WrapperFunctionResult)>;
diff --git a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
index 6889c7e642ec69ad747004d8eff64aacd5653f42..707239c7ba36452e09d2fe2f87a79576b954da18 100644
--- a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
+++ b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
@@ -203,6 +203,6 @@ class Directive<string d> {
   // What the directive is associated with.
   Association association = AS_FromLeaves;
 
-  // The category if the directive.
+  // The category of the directive.
   Category category = ?;
 }
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 30a81efe7f8a627ffef2ed0ce5be3e3f7afe3063..cda1d96b2a8cf4c4ad3a64cf02c2747ffbe24e3b 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -310,7 +310,7 @@ def ACC_Declare : Directive<"declare"> {
     VersionedClause<ACCC_Link>
   ];
   let association = AS_None;
-  let category = CA_Executable;
+  let category = CA_Declarative;
 }
 
 // 2.5.3
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index a05d1a4d587845c13fc3d989c569d3a3beeb09fd..55e32028e3ed081d0f09c94017b83e23466596cd 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -267,6 +267,9 @@ namespace CallingConv {
     /// Calling convention used for RISC-V V-extension.
     RISCV_VectorCall = 110,
 
+    /// Preserve X1-X15, X19-X29, SP, Z0-Z31, P0-P15.
+    AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 = 111,
+
     /// The highest possible ID. Must be some 2^k - 1.
     MaxID = 1023
   };
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 18873a551595aebca8899c235db8531dde6b95a0..c15d64b8422938a43975edbf1fe9c80fc96cf918 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -2147,13 +2147,20 @@ public:
   static unsigned
   getBaseDiscriminatorFromDiscriminator(unsigned D,
                                         bool IsFSDiscriminator = false) {
-    // Return the probe id instead of zero for a pseudo probe discriminator.
-    // This should help differenciate callsites with same line numbers to
-    // achieve a decent AutoFDO profile under -fpseudo-probe-for-profiling,
-    // where the original callsite dwarf discriminator is overwritten by
-    // callsite probe information.
-    if (isPseudoProbeDiscriminator(D))
+    // Extract the dwarf base discriminator if it's encoded in the pseudo probe
+    // discriminator.
+    if (isPseudoProbeDiscriminator(D)) {
+      auto DwarfBaseDiscriminator =
+          PseudoProbeDwarfDiscriminator::extractDwarfBaseDiscriminator(D);
+      if (DwarfBaseDiscriminator)
+        return *DwarfBaseDiscriminator;
+      // Return the probe id instead of zero for a pseudo probe discriminator.
+      // This should help differenciate callsites with same line numbers to
+      // achieve a decent AutoFDO profile under -fpseudo-probe-for-profiling,
+      // where the original callsite dwarf discriminator is overwritten by
+      // callsite probe information.
       return PseudoProbeDwarfDiscriminator::extractProbeIndex(D);
+    }
 
     if (IsFSDiscriminator)
       return getMaskedDiscriminator(D, getBaseDiscriminatorBits());
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index 3c42eb2cf2a5aacfb3b134c76dad9f84450af8e5..921001c8a5d512852ef88debb32c896ca2177812 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_IRBUILDERFOLDER_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/GEPNoWrapFlags.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 9dd1bb455a718ed137916aefec62b8721bab581e..ad649b53761a3334f225fb7b837a510ace4c03d6 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1588,6 +1588,14 @@ public:
   static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID,
                                        BasicBlock::iterator InsertPt);
 
+  /// Return the convergence control token for this call, if it exists.
+  Value *getConvergenceControlToken() const {
+    if (auto Bundle = getOperandBundle(llvm::LLVMContext::OB_convergencectrl)) {
+      return Bundle->Inputs[0].get();
+    }
+    return nullptr;
+  }
+
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Call ||
            I->getOpcode() == Instruction::Invoke ||
@@ -2120,6 +2128,15 @@ public:
     return Attrs.getParamStackAlignment(ArgNo);
   }
 
+  /// Extract the byref type for a call or parameter.
+  Type *getParamByRefType(unsigned ArgNo) const {
+    if (auto *Ty = Attrs.getParamByRefType(ArgNo))
+      return Ty;
+    if (const Function *F = getCalledFunction())
+      return F->getAttributes().getParamByRefType(ArgNo);
+    return nullptr;
+  }
+
   /// Extract the byval type for a call or parameter.
   Type *getParamByValType(unsigned ArgNo) const {
     if (auto *Ty = Attrs.getParamByValType(ArgNo))
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index fcd3a1025ac135860a498c670dbb418b9b26edde..9010e1a1c896bfba1977eb1cd8e609695369df3e 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -1799,17 +1799,14 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  // Returns the convergence intrinsic referenced by |I|'s convergencectrl
-  // attribute if any.
-  static IntrinsicInst *getParentConvergenceToken(Instruction *I) {
-    auto *CI = dyn_cast<llvm::CallInst>(I);
-    if (!CI)
-      return nullptr;
-
-    auto Bundle = CI->getOperandBundle(llvm::LLVMContext::OB_convergencectrl);
-    assert(Bundle->Inputs.size() == 1 &&
-           Bundle->Inputs[0]->getType()->isTokenTy());
-    return dyn_cast<llvm::IntrinsicInst>(Bundle->Inputs[0].get());
+  bool isAnchor() {
+    return getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
+  }
+  bool isEntry() {
+    return getIntrinsicID() == Intrinsic::experimental_convergence_entry;
+  }
+  bool isLoop() {
+    return getIntrinsicID() == Intrinsic::experimental_convergence_loop;
   }
 };
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index bfb966dfad02db6cb20e9c908951c65a8740abe0..4db98e1b31d9ce99cc42a758699ecbec4965dafc 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1083,18 +1083,6 @@ def int_amdgcn_make_buffer_rsrc : DefaultAttrsIntrinsic <
 
 defset list<AMDGPURsrcIntrinsic> AMDGPUBufferIntrinsics = {
 
-class AMDGPUBufferLoad<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
-  [data_ty],
-  [llvm_v4i32_ty,     // rsrc(SGPR)
-   llvm_i32_ty,       // vindex(VGPR)
-   llvm_i32_ty,       // offset(SGPR/VGPR/imm)
-   llvm_i1_ty,        // glc(imm)
-   llvm_i1_ty],       // slc(imm)
-  [IntrReadMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<0>;
-def int_amdgcn_buffer_load_format : AMDGPUBufferLoad<llvm_anyfloat_ty>;
-def int_amdgcn_buffer_load : AMDGPUBufferLoad;
-
 // Generate a buffer_load instruction that may be optimized to s_buffer_load if
 // the offset argument is uniform.
 def int_amdgcn_s_buffer_load : DefaultAttrsIntrinsic <
@@ -1111,25 +1099,12 @@ def int_amdgcn_s_buffer_load : DefaultAttrsIntrinsic <
   [IntrNoMem, ImmArg<ArgIndex<2>>]>,
   AMDGPURsrcIntrinsic<0>;
 
-class AMDGPUBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsIntrinsic <
-  [],
-  [data_ty,          // vdata(VGPR)
-   llvm_v4i32_ty,     // rsrc(SGPR)
-   llvm_i32_ty,       // vindex(VGPR)
-   llvm_i32_ty,       // offset(SGPR/VGPR/imm)
-   llvm_i1_ty,        // glc(imm)
-   llvm_i1_ty],       // slc(imm)
-  [IntrWriteMem, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<1>;
-def int_amdgcn_buffer_store_format : AMDGPUBufferStore<llvm_anyfloat_ty>;
-def int_amdgcn_buffer_store : AMDGPUBufferStore;
-
-// New buffer intrinsics with separate raw and struct variants.  The raw
+// Buffer intrinsics with separate raw and struct variants.  The raw
 // variant never has an index. The struct variant always has an index, even if
 // it is const 0. A struct intrinsic with constant 0 index is different to the
 // corresponding raw intrinsic on gfx9+ because the behavior of bound checking
 // and swizzling changes depending on whether idxen is set in the instruction.
-// These new instrinsics also keep the offset and soffset arguments separate as
+// These instrinsics also keep the offset and soffset arguments separate as
 // they behave differently in bounds checking and swizzling.
 
 // The versions of these intrinsics that take <4 x i32> arguments are deprecated
@@ -1489,41 +1464,7 @@ def int_amdgcn_struct_buffer_atomic_fmax : AMDGPUStructBufferAtomic<llvm_anyfloa
 def int_amdgcn_struct_ptr_buffer_atomic_fmin : AMDGPUStructPtrBufferAtomic<llvm_anyfloat_ty>;
 def int_amdgcn_struct_ptr_buffer_atomic_fmax : AMDGPUStructPtrBufferAtomic<llvm_anyfloat_ty>;
 
-// Obsolescent tbuffer intrinsics.
-def int_amdgcn_tbuffer_load : DefaultAttrsIntrinsic <
-    [llvm_any_ty],    // overloaded for types f32/i32, v2f32/v2i32, v4f32/v4i32
-    [llvm_v4i32_ty,   // rsrc(SGPR)
-     llvm_i32_ty,     // vindex(VGPR)
-     llvm_i32_ty,     // voffset(VGPR)
-     llvm_i32_ty,     // soffset(SGPR)
-     llvm_i32_ty,     // offset(imm)
-     llvm_i32_ty,     // dfmt(imm)
-     llvm_i32_ty,     // nfmt(imm)
-     llvm_i1_ty,     // glc(imm)
-     llvm_i1_ty],    // slc(imm)
-    [IntrReadMem,
-     ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>, ImmArg<ArgIndex<6>>,
-     ImmArg<ArgIndex<7>>, ImmArg<ArgIndex<8>>], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<0>;
-
-def int_amdgcn_tbuffer_store : DefaultAttrsIntrinsic <
-    [],
-    [llvm_any_ty,    // vdata(VGPR), overloaded for types f32/i32, v2f32/v2i32, v4f32/v4i32
-     llvm_v4i32_ty,  // rsrc(SGPR)
-     llvm_i32_ty,    // vindex(VGPR)
-     llvm_i32_ty,    // voffset(VGPR)
-     llvm_i32_ty,    // soffset(SGPR)
-     llvm_i32_ty,    // offset(imm)
-     llvm_i32_ty,    // dfmt(imm)
-     llvm_i32_ty,    // nfmt(imm)
-     llvm_i1_ty,     // glc(imm)
-     llvm_i1_ty],    // slc(imm)
-    [IntrWriteMem, ImmArg<ArgIndex<5>>,
-     ImmArg<ArgIndex<6>>, ImmArg<ArgIndex<7>>,
-     ImmArg<ArgIndex<8>>, ImmArg<ArgIndex<9>>], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<1>;
-
-// New tbuffer intrinsics, with:
+// tbuffer intrinsics, with:
 // - raw and struct variants
 // - joint format field
 // - joint cachepolicy field
@@ -1670,51 +1611,6 @@ def int_amdgcn_struct_tbuffer_store : DefaultAttrsIntrinsic <
      ImmArg<ArgIndex<5>>, ImmArg<ArgIndex<6>>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<1>;
 
-class AMDGPUBufferAtomic : Intrinsic <
-  [llvm_anyint_ty],
-  [LLVMMatchType<0>,       // vdata(VGPR)
-   llvm_v4i32_ty,     // rsrc(SGPR)
-   llvm_i32_ty,       // vindex(VGPR)
-   llvm_i32_ty,       // offset(SGPR/VGPR/imm)
-   llvm_i1_ty],       // slc(imm)
-  [ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<1, 0>;
-def int_amdgcn_buffer_atomic_swap : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_add : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_sub : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_smin : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_umin : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_smax : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_umax : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_and : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_or : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_xor : AMDGPUBufferAtomic;
-def int_amdgcn_buffer_atomic_cmpswap : Intrinsic<
-  [llvm_i32_ty],
-  [llvm_i32_ty,       // src(VGPR)
-   llvm_i32_ty,       // cmp(VGPR)
-   llvm_v4i32_ty,     // rsrc(SGPR)
-   llvm_i32_ty,       // vindex(VGPR)
-   llvm_i32_ty,       // offset(SGPR/VGPR/imm)
-   llvm_i1_ty],       // slc(imm)
-  [ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<2, 0>;
-
-def int_amdgcn_buffer_atomic_csub : AMDGPUBufferAtomic;
-
-class AMDGPUBufferAtomicFP : Intrinsic <
-  [llvm_anyfloat_ty],
-  [LLVMMatchType<0>, // vdata(VGPR)
-   llvm_v4i32_ty,    // rsrc(SGPR)
-   llvm_i32_ty,      // vindex(VGPR)
-   llvm_i32_ty,      // offset(SGPR/VGPR/imm)
-   llvm_i1_ty],      // slc(imm)
-  [ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
-  AMDGPURsrcIntrinsic<1, 0>;
-
-// Legacy form of the intrinsic. raw and struct forms should be preferred.
-def int_amdgcn_buffer_atomic_fadd : AMDGPUBufferAtomicFP;
-
 class AMDGPURawBufferLoadLDS : Intrinsic <
   [],
   [llvm_v4i32_ty,             // rsrc(SGPR)
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index fda26891acfa73da2ea35bb700ca829e62de089d..f63f54ef941076be3114dab470d87e5a16567129 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -330,8 +330,6 @@ public:
     unsigned Opcode;
     if (auto *I = dyn_cast<Instruction>(V))
       Opcode = I->getOpcode();
-    else if (auto *CE = dyn_cast<ConstantExpr>(V))
-      Opcode = CE->getOpcode();
     else
       return false;
 
diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h
index cdbd498a8be61ac24d93c9ea5647f0e204cf8bb2..7308904da3fd3007b1d16ef335614667b7a20089 100644
--- a/llvm/include/llvm/IR/PseudoProbe.h
+++ b/llvm/include/llvm/IR/PseudoProbe.h
@@ -44,26 +44,51 @@ public:
   // 32-bit integer which is organized as:
   //  [2:0] - 0x7, this is reserved for regular discriminator,
   //          see DWARF discriminator encoding rule
-  //  [18:3] - probe id
+  //  if the [28:28] bit is zero:
+  //    [18:3] for probe id.
+  //  else:
+  //    [15:3] for probe id, [18:16] for dwarf base discriminator.
   //  [25:19] - probe distribution factor
-  //  [28:26] - probe type, see PseudoProbeType
-  //  [31:29] - reserved for probe attributes
-  static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags,
-                                uint32_t Factor) {
+  //  [27:26] - probe type, see PseudoProbeType
+  //  [28:28] - indicates whether dwarf base discriminator is encoded.
+  //  [30:29] - reserved for probe attributes
+  static uint32_t
+  packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, uint32_t Factor,
+                std::optional<uint32_t> DwarfBaseDiscriminator) {
     assert(Index <= 0xFFFF && "Probe index too big to encode, exceeding 2^16");
-    assert(Type <= 0x7 && "Probe type too big to encode, exceeding 7");
+    assert(Type <= 0x3 && "Probe type too big to encode, exceeding 3");
     assert(Flags <= 0x7);
     assert(Factor <= 100 &&
            "Probe distribution factor too big to encode, exceeding 100");
-    return (Index << 3) | (Factor << 19) | (Type << 26) | 0x7;
+    uint32_t V = (Index << 3) | (Factor << 19) | (Type << 26) | 0x7;
+    // If both the probe id and dwarf base discriminator is small, the probe id
+    // space is shared with the dwarf base discriminator, this is to make the
+    // probe-based build compatible with the dwarf-based profile.
+    // Pack the dwarf base discriminator into [18:16] and set the [28:28] bit.
+    if (Index <= 0x1FFF && DwarfBaseDiscriminator &&
+        *DwarfBaseDiscriminator <= 0x7)
+      V |= (1 << 28) | (*DwarfBaseDiscriminator << 16);
+    return V;
   }
 
   static uint32_t extractProbeIndex(uint32_t Value) {
+    if (isDwarfBaseDiscriminatorEncoded(Value))
+      return (Value >> 3) & 0x1FFF;
     return (Value >> 3) & 0xFFFF;
   }
 
+  static std::optional<uint32_t> extractDwarfBaseDiscriminator(uint32_t Value) {
+    if (isDwarfBaseDiscriminatorEncoded(Value))
+      return (Value >> 16) & 0x7;
+    return std::nullopt;
+  }
+
+  static bool isDwarfBaseDiscriminatorEncoded(uint32_t Value) {
+    return Value & 0x10000000;
+  }
+
   static uint32_t extractProbeType(uint32_t Value) {
-    return (Value >> 26) & 0x7;
+    return (Value >> 26) & 0x3;
   }
 
   static uint32_t extractProbeAttributes(uint32_t Value) {
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index a5a72884a6bf51f31cad9aea7e736a02bfced519..e900bcdbcdd0498022f0741b6cffaac3ff7179c0 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -197,6 +197,11 @@ HANDLE_LIBCALL(COS_F64, "cos")
 HANDLE_LIBCALL(COS_F80, "cosl")
 HANDLE_LIBCALL(COS_F128, "cosl")
 HANDLE_LIBCALL(COS_PPCF128, "cosl")
+HANDLE_LIBCALL(TAN_F32, "tanf")
+HANDLE_LIBCALL(TAN_F64, "tan")
+HANDLE_LIBCALL(TAN_F80, "tanl")
+HANDLE_LIBCALL(TAN_F128,"tanl")
+HANDLE_LIBCALL(TAN_PPCF128, "tanl")
 HANDLE_LIBCALL(SINCOS_F32, nullptr)
 HANDLE_LIBCALL(SINCOS_F64, nullptr)
 HANDLE_LIBCALL(SINCOS_F80, nullptr)
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index c4c1825bbf09ebaa07f77be1c1c79d9c29fc3073..8803ef5a90e6e4b9b93bcae1451fc4d2f8fa2059 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -106,6 +106,7 @@ void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&);
 void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
 void initializeExpandPostRAPass(PassRegistry&);
 void initializeExpandReductionsPass(PassRegistry&);
+void initializeExpandVariadicsPass(PassRegistry &);
 void initializeExpandVectorPredicationPass(PassRegistry &);
 void initializeExternalAAWrapperPassPass(PassRegistry&);
 void initializeFEntryInserterPass(PassRegistry&);
diff --git a/llvm/include/llvm/MC/MCAsmLayout.h b/llvm/include/llvm/MC/MCAsmLayout.h
index 94cfb76bdce4db7da5fd94d77c55874b8674d9cb..6fbfce78759e1b85c2c86b40f4f800442380b1bc 100644
--- a/llvm/include/llvm/MC/MCAsmLayout.h
+++ b/llvm/include/llvm/MC/MCAsmLayout.h
@@ -31,37 +31,21 @@ class MCAsmLayout {
   /// List of sections in layout order.
   llvm::SmallVector<MCSection *, 16> SectionOrder;
 
-  /// The last fragment which was laid out, or 0 if nothing has been laid
-  /// out. Fragments are always laid out in order, so all fragments with a
-  /// lower ordinal will be valid.
-  mutable DenseMap<const MCSection *, MCFragment *> LastValidFragment;
-
-  /// Make sure that the layout for the given fragment is valid, lazily
-  /// computing it if necessary.
+  /// Compute the layout for the section if necessary.
   void ensureValid(const MCFragment *F) const;
 
-  /// Is the layout for this fragment valid?
-  bool isFragmentValid(const MCFragment *F) const;
-
 public:
   MCAsmLayout(MCAssembler &Assembler);
 
   /// Get the assembler object this is a layout for.
   MCAssembler &getAssembler() const { return Assembler; }
 
-  /// \returns whether the offset of fragment \p F can be obtained via
-  /// getFragmentOffset.
-  bool canGetFragmentOffset(const MCFragment *F) const;
-
   /// Invalidate the fragments starting with F because it has been
   /// resized. The fragment's size should have already been updated, but
   /// its bundle padding will be recomputed.
   void invalidateFragmentsFrom(MCFragment *F);
 
-  /// Perform layout for a single fragment, assuming that the previous
-  /// fragment has already been laid out correctly, and the parent section has
-  /// been initialized.
-  void layoutFragment(MCFragment *Fragment);
+  void layoutBundle(MCFragment *F);
 
   /// \name Section Access (in layout order)
   /// @{
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 3fa5f2fe655e86e5f0b98a35eb6a65366e415c02..914c7506e754b393d66d2b6c1085d1c55f2b1f9f 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -203,10 +203,6 @@ private:
   /// were adjusted.
   bool layoutOnce(MCAsmLayout &Layout);
 
-  /// Perform one layout iteration of the given section and return true
-  /// if any offsets were adjusted.
-  bool layoutSectionOnce(MCAsmLayout &Layout, MCSection &Sec);
-
   /// Perform relaxation on a single fragment - returns true if the fragment
   /// changes as a result of relaxation.
   bool relaxFragment(MCAsmLayout &Layout, MCFragment &F);
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index a9b19dc56f16a5e9dd09b97be5a1f57a49047a51..67e10c35894954aea794d30cc0f1e65ede294865 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -70,9 +70,6 @@ private:
 
   FragmentType Kind;
 
-  /// Whether fragment is being laid out.
-  bool IsBeingLaidOut;
-
 protected:
   bool HasInstructions;
   bool LinkerRelaxable = false;
diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h
index 2bc310852fe50d6438e42f5606f1e51660e4d522..578b7328970b760441f1180e90056058018d6b4f 100644
--- a/llvm/include/llvm/MC/MCInst.h
+++ b/llvm/include/llvm/MC/MCInst.h
@@ -189,7 +189,7 @@ class MCInst {
   unsigned Flags = 0;
 
   SMLoc Loc;
-  SmallVector<MCOperand, 10> Operands;
+  SmallVector<MCOperand, 6> Operands;
 
 public:
   MCInst() = default;
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 90bc48ec185ca4afae8bf6afe6a25ecb45812f73..90effde5bb670a5df64f2d3f1fe115d966b7655b 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -89,6 +89,8 @@ private:
   /// Whether this section has had instructions emitted into it.
   bool HasInstructions : 1;
 
+  bool HasLayout : 1;
+
   bool IsRegistered : 1;
 
   MCDummyFragment DummyFragment;
@@ -166,6 +168,9 @@ public:
   bool hasInstructions() const { return HasInstructions; }
   void setHasInstructions(bool Value) { HasInstructions = Value; }
 
+  bool hasLayout() const { return HasLayout; }
+  void setHasLayout(bool Value) { HasLayout = Value; }
+
   bool isRegistered() const { return IsRegistered; }
   void setIsRegistered(bool Value) { IsRegistered = Value; }
 
@@ -188,6 +193,8 @@ public:
   iterator end() { return Fragments.end(); }
   const_iterator end() const { return Fragments.end(); }
 
+  void addFragment(MCFragment &F) { Fragments.push_back(&F); }
+
   MCSection::iterator getSubsectionInsertionPoint(unsigned Subsection);
 
   void dump() const;
diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h
index 0ce95c72a73f7026dd3b015915483a158b336a0a..0c2b97a594231b3516b20e5521c97bcadcb8e1d4 100644
--- a/llvm/include/llvm/MC/MCSymbolWasm.h
+++ b/llvm/include/llvm/MC/MCSymbolWasm.h
@@ -114,9 +114,11 @@ public:
     return isTable() && hasTableType() &&
            getTableType().ElemType == wasm::ValType::FUNCREF;
   }
-  void setFunctionTable() {
+  void setFunctionTable(bool is64) {
     setType(wasm::WASM_SYMBOL_TYPE_TABLE);
-    setTableType(wasm::ValType::FUNCREF);
+    uint8_t flags =
+        is64 ? wasm::WASM_LIMITS_FLAG_IS_64 : wasm::WASM_LIMITS_FLAG_NONE;
+    setTableType(wasm::ValType::FUNCREF, flags);
   }
 
   void setUsedInGOT() const { IsUsedInGOT = true; }
@@ -140,10 +142,11 @@ public:
     return *TableType;
   }
   void setTableType(wasm::WasmTableType TT) { TableType = TT; }
-  void setTableType(wasm::ValType VT) {
+  void setTableType(wasm::ValType VT,
+                    uint8_t flags = wasm::WASM_LIMITS_FLAG_NONE) {
     // Declare a table with element type VT and no limits (min size 0, no max
     // size).
-    wasm::WasmLimits Limits = {wasm::WASM_LIMITS_FLAG_NONE, 0, 0};
+    wasm::WasmLimits Limits = {flags, 0, 0};
     setTableType({VT, Limits});
   }
 };
diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 0b3bea786d31c70bc53b718a99140e507bc7ede6..7b65e325ef05d2746a7c0836ff3e5391883c4b18 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -522,7 +522,7 @@ template <> struct ScalarEnumerationTraits<dwarf::LoclistEntries> {
   }
 };
 
-#define HANDLE_DW_OP(id, name, version, vendor)                                \
+#define HANDLE_DW_OP(id, name, operands, arity, version, vendor)               \
   io.enumCase(value, "DW_OP_" #name, dwarf::DW_OP_##name);
 
 template <> struct ScalarEnumerationTraits<dwarf::LocationAtom> {
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index 7542986837ec29c374e8b3f9d4588bcdb0ad25e1..eef1709bce5dfc81c8027eb6a4d8fb97d507c26f 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -43,6 +43,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/RegAllocFast.h"
 #include "llvm/CodeGen/ReplaceWithVeclib.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
@@ -1038,7 +1039,7 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addTargetRegisterAllocator(
   if (Optimized)
     addPass(RAGreedyPass());
   else
-    addPass(RAFastPass());
+    addPass(RegAllocFastPass());
 }
 
 /// Find and instantiate the register allocation pass requested by this target
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 4152c35ded6cf620479fc93b97f4085d7cbaa17d..e59591e473b79cf9986d500e3c4a3f23c1ae7e7b 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -133,6 +133,19 @@ MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
 MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
 #undef MACHINE_FUNCTION_PASS
 
+#ifndef MACHINE_FUNCTION_PASS_WITH_PARAMS
+#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER,    \
+                                          PARAMS)
+#endif
+MACHINE_FUNCTION_PASS_WITH_PARAMS(
+    "regallocfast", "RegAllocFast",
+    [](RegAllocFastPassOptions Opts) { return RegAllocFastPass(Opts); },
+    [PB = this](StringRef Params) {
+      return parseRegAllocFastPassOptions(*PB, Params);
+    },
+    "filter=reg-filter;no-clear-vregs")
+#undef MACHINE_FUNCTION_PASS_WITH_PARAMS
+
 // After a pass is converted to new pass manager, its entry should be moved from
 // dummy table to the normal one. For example, for a machine function pass,
 // DUMMY_MACHINE_FUNCTION_PASS to MACHINE_FUNCTION_PASS.
@@ -211,7 +224,6 @@ DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass)
 DUMMY_MACHINE_FUNCTION_PASS("prologepilog", PrologEpilogInserterPass)
 DUMMY_MACHINE_FUNCTION_PASS("prologepilog-code", PrologEpilogCodeInserterPass)
 DUMMY_MACHINE_FUNCTION_PASS("ra-basic", RABasicPass)
-DUMMY_MACHINE_FUNCTION_PASS("ra-fast", RAFastPass)
 DUMMY_MACHINE_FUNCTION_PASS("ra-greedy", RAGreedyPass)
 DUMMY_MACHINE_FUNCTION_PASS("ra-pbqp", RAPBQPPass)
 DUMMY_MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass)
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 8de0f024a4b115cb48d5bc0abb793f408a33cc84..ed817127c3db11a4bc4cb23db27b6b0afa421128 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -17,6 +17,7 @@
 
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/RegAllocCommon.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Passes/OptimizationLevel.h"
 #include "llvm/Support/Error.h"
@@ -388,6 +389,9 @@ public:
   /// returns false.
   Error parseAAPipeline(AAManager &AA, StringRef PipelineText);
 
+  /// Parse RegClassFilterName to get RegClassFilterFunc.
+  RegClassFilterFunc parseRegAllocFilter(StringRef RegClassFilterName);
+
   /// Print pass names.
   void printPassNames(raw_ostream &OS);
 
@@ -576,6 +580,14 @@ public:
   }
   /// @}}
 
+  /// Register callbacks to parse target specific filter field if regalloc pass
+  /// needs it. E.g. AMDGPU requires regalloc passes can handle sgpr and vgpr
+  /// separately.
+  void registerRegClassFilterParsingCallback(
+      const std::function<RegClassFilterFunc(StringRef)> &C) {
+    RegClassFilterParsingCallbacks.push_back(C);
+  }
+
   /// Register a callback for a top-level pipeline entry.
   ///
   /// If the PassManager type is not given at the top level of the pipeline
@@ -792,6 +804,9 @@ private:
                                  ArrayRef<PipelineElement>)>,
               2>
       MachineFunctionPipelineParsingCallbacks;
+  // Callbacks to parse `filter` parameter in register allocation passes
+  SmallVector<std::function<RegClassFilterFunc(StringRef)>, 2>
+      RegClassFilterParsingCallbacks;
 };
 
 /// This utility template takes care of adding require<> and invalidate<>
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 84d1b541171bf1c208bae4587727b067c69eac32..9de8ba5b8bf5128fe240dfd01cbbde0345ebb137 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -171,7 +171,7 @@ public:
                     FunctionAnalysisManager::Invalidator &);
   };
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   SmallVector<StringRef, 8> PassStack;
 #endif
 
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 15b9eb688e27e5fe3de9430f1e9de52e9437db18..d6831eeaa794b7520a1575b917bfb115e76ee921 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -729,23 +729,20 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
 }
 
 GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) {
-  if (auto Iter = MD5VTableMap.find(MD5Hash); Iter != MD5VTableMap.end())
-    return Iter->second;
-  return nullptr;
+  return MD5VTableMap.lookup(MD5Hash);
 }
 
 // To store the sums of profile count values, or the percentage of
 // the sums of the total count values.
 struct CountSumOrPercent {
-  uint64_t NumEntries;
-  double CountSum;
-  double ValueCounts[IPVK_Last - IPVK_First + 1];
-  CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {}
+  uint64_t NumEntries = 0;
+  double CountSum = 0.0f;
+  std::array<double, IPVK_Last - IPVK_First + 1> ValueCounts = {};
+  CountSumOrPercent() = default;
   void reset() {
     NumEntries = 0;
     CountSum = 0.0f;
-    for (double &VC : ValueCounts)
-      VC = 0.0f;
+    ValueCounts.fill(0.0f);
   }
 };
 
@@ -761,15 +758,13 @@ struct OverlapStats {
   CountSumOrPercent Mismatch;
   CountSumOrPercent Unique;
   OverlapStatsLevel Level;
-  const std::string *BaseFilename;
-  const std::string *TestFilename;
+  const std::string *BaseFilename = nullptr;
+  const std::string *TestFilename = nullptr;
   StringRef FuncName;
-  uint64_t FuncHash;
-  bool Valid;
+  uint64_t FuncHash = 0;
+  bool Valid = false;
 
-  OverlapStats(OverlapStatsLevel L = ProgramLevel)
-      : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0),
-        Valid(false) {}
+  OverlapStats(OverlapStatsLevel L = ProgramLevel) : Level(L) {}
 
   void dump(raw_fd_ostream &OS) const;
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 1c3d4c795f631dcf6994d81168e423a9655c3e1a..3b307d083599806e1b88f457a5f16f79ab41b68e 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -665,9 +665,8 @@ private:
   const unsigned char *CallStackBase = nullptr;
 
   Error deserializeV012(const unsigned char *Start, const unsigned char *Ptr,
-                        uint64_t FirstWord, memprof::IndexedVersion Version);
-  Error deserializeV3(const unsigned char *Start, const unsigned char *Ptr,
-                      memprof::IndexedVersion Version);
+                        uint64_t FirstWord);
+  Error deserializeV3(const unsigned char *Start, const unsigned char *Ptr);
 
 public:
   IndexedMemProfReader() = default;
@@ -694,22 +693,15 @@ private:
   /// Context sensitive profile summary data.
   std::unique_ptr<ProfileSummary> CS_Summary;
   IndexedMemProfReader MemProfReader;
-  /// VTableNamePtr points to the beginning of compressed vtable names.
-  /// When a symtab is constructed from profiles by llvm-profdata, the list of
-  /// names could be decompressed based on `VTableNamePtr` and
-  /// `CompressedVTableNamesLen`.
+  /// The compressed vtable names, to be used for symtab construction.
   /// A compiler that reads indexed profiles could construct symtab from module
   /// IR so it doesn't need the decompressed names.
-  const char *VTableNamePtr = nullptr;
-  /// The length of compressed vtable names.
-  uint64_t CompressedVTableNamesLen = 0;
-  /// Total size of binary ids.
-  uint64_t BinaryIdsSize{0};
-  /// Start address of binary id length and data pairs.
-  const uint8_t *BinaryIdsStart = nullptr;
+  StringRef VTableName;
+  /// A memory buffer holding binary ids.
+  ArrayRef<uint8_t> BinaryIdsBuffer;
 
   // Index to the current record in the record array.
-  unsigned RecordIndex;
+  unsigned RecordIndex = 0;
 
   // Read the profile summary. Return a pointer pointing to one byte past the
   // end of the summary data if it exists or the input \c Cur.
@@ -722,7 +714,7 @@ public:
       std::unique_ptr<MemoryBuffer> DataBuffer,
       std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr)
       : DataBuffer(std::move(DataBuffer)),
-        RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {}
+        RemappingBuffer(std::move(RemappingBuffer)) {}
   IndexedInstrProfReader(const IndexedInstrProfReader &) = delete;
   IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete;
 
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 406144d9db1e857d12bbc14352d2154469018eb0..53ddfd192341086f5aef587dc6bd1d98856e875c 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -28,7 +28,8 @@ enum IndexedVersion : uint64_t {
   Version1 = 1,
   // Version 2: Added a call stack table.
   Version2 = 2,
-  // Version 3: Under development.
+  // Version 3: Added a radix tree for call stacks.  Switched to linear IDs for
+  // frames and call stacks.
   Version3 = 3,
 };
 
@@ -199,7 +200,7 @@ struct Frame {
   GlobalValue::GUID Function;
   // The symbol name for the function. Only populated in the Frame by the reader
   // if requested during initialization. This field should not be serialized.
-  std::optional<std::string> SymbolName;
+  std::unique_ptr<std::string> SymbolName;
   // The source line offset of the call from the beginning of parent function.
   uint32_t LineOffset;
   // The source column number of the call to help distinguish multiple calls
@@ -210,7 +211,9 @@ struct Frame {
 
   Frame(const Frame &Other) {
     Function = Other.Function;
-    SymbolName = Other.SymbolName;
+    SymbolName = Other.SymbolName
+                     ? std::make_unique<std::string>(*Other.SymbolName)
+                     : nullptr;
     LineOffset = Other.LineOffset;
     Column = Other.Column;
     IsInlineFrame = Other.IsInlineFrame;
@@ -228,7 +231,9 @@ struct Frame {
 
   Frame &operator=(const Frame &Other) {
     Function = Other.Function;
-    SymbolName = Other.SymbolName;
+    SymbolName = Other.SymbolName
+                     ? std::make_unique<std::string>(*Other.SymbolName)
+                     : nullptr;
     LineOffset = Other.LineOffset;
     Column = Other.Column;
     IsInlineFrame = Other.IsInlineFrame;
@@ -237,10 +242,10 @@ struct Frame {
 
   bool operator!=(const Frame &Other) const { return !operator==(Other); }
 
-  bool hasSymbolName() const { return SymbolName.has_value(); }
+  bool hasSymbolName() const { return !!SymbolName; }
 
   StringRef getSymbolName() const {
-    assert(SymbolName.has_value());
+    assert(hasSymbolName());
     return *SymbolName;
   }
 
@@ -364,7 +369,7 @@ struct IndexedAllocationInfo {
 // be used for temporary in-memory instances.
 struct AllocationInfo {
   // Same as IndexedAllocationInfo::CallStack with the frame contents inline.
-  llvm::SmallVector<Frame> CallStack;
+  std::vector<Frame> CallStack;
   // Same as IndexedAllocationInfo::Info;
   PortableMemInfoBlock Info;
 
@@ -435,8 +440,8 @@ struct IndexedMemProfRecord {
   // on the schema provided in \p Schema.
   void serialize(const MemProfSchema &Schema, raw_ostream &OS,
                  IndexedVersion Version,
-                 llvm::DenseMap<memprof::CallStackId, LinearCallStackId>
-                     *MemProfCallStackIndexes = nullptr);
+                 llvm::DenseMap<CallStackId, LinearCallStackId>
+                     *MemProfCallStackIndexes = nullptr) const;
 
   // Deserializes memprof records from the Buffer.
   static IndexedMemProfRecord deserialize(const MemProfSchema &Schema,
@@ -446,8 +451,7 @@ struct IndexedMemProfRecord {
   // Convert IndexedMemProfRecord to MemProfRecord.  Callback is used to
   // translate CallStackId to call stacks with frames inline.
   MemProfRecord toMemProfRecord(
-      llvm::function_ref<llvm::SmallVector<Frame>(const CallStackId)> Callback)
-      const;
+      llvm::function_ref<std::vector<Frame>(const CallStackId)> Callback) const;
 
   // Returns the GUID for the function name after canonicalization. For
   // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are
@@ -462,7 +466,7 @@ struct MemProfRecord {
   // Same as IndexedMemProfRecord::AllocSites with frame contents inline.
   llvm::SmallVector<AllocationInfo> AllocSites;
   // Same as IndexedMemProfRecord::CallSites with frame contents inline.
-  llvm::SmallVector<llvm::SmallVector<Frame>> CallSites;
+  llvm::SmallVector<std::vector<Frame>> CallSites;
 
   MemProfRecord() = default;
   MemProfRecord(
@@ -472,7 +476,7 @@ struct MemProfRecord {
       AllocSites.emplace_back(IndexedAI, IdToFrameCallback);
     }
     for (const ArrayRef<FrameId> Site : Record.CallSites) {
-      llvm::SmallVector<Frame> Frames;
+      std::vector<Frame> Frames;
       for (const FrameId Id : Site) {
         Frames.push_back(IdToFrameCallback(Id));
       }
@@ -490,7 +494,7 @@ struct MemProfRecord {
 
     if (!CallSites.empty()) {
       OS << "    CallSites:\n";
-      for (const llvm::SmallVector<Frame> &Frames : CallSites) {
+      for (const std::vector<Frame> &Frames : CallSites) {
         for (const Frame &F : Frames) {
           OS << "    -\n";
           F.printYAML(OS);
@@ -576,15 +580,14 @@ private:
   IndexedVersion Version;
 
   // Mappings from CallStackId to the indexes into the call stack array.
-  llvm::DenseMap<memprof::CallStackId, LinearCallStackId>
-      *MemProfCallStackIndexes;
+  llvm::DenseMap<CallStackId, LinearCallStackId> *MemProfCallStackIndexes;
 
 public:
   // We do not support the default constructor, which does not set Version.
   RecordWriterTrait() = delete;
-  RecordWriterTrait(const MemProfSchema *Schema, IndexedVersion V,
-                    llvm::DenseMap<memprof::CallStackId, LinearCallStackId>
-                        *MemProfCallStackIndexes)
+  RecordWriterTrait(
+      const MemProfSchema *Schema, IndexedVersion V,
+      llvm::DenseMap<CallStackId, LinearCallStackId> *MemProfCallStackIndexes)
       : Schema(Schema), Version(V),
         MemProfCallStackIndexes(MemProfCallStackIndexes) {}
 
@@ -844,8 +847,8 @@ template <typename MapTy> struct CallStackIdConverter {
   CallStackIdConverter(const CallStackIdConverter &) = delete;
   CallStackIdConverter &operator=(const CallStackIdConverter &) = delete;
 
-  llvm::SmallVector<Frame> operator()(CallStackId CSId) {
-    llvm::SmallVector<Frame> Frames;
+  std::vector<Frame> operator()(CallStackId CSId) {
+    std::vector<Frame> Frames;
     auto CSIter = Map.find(CSId);
     if (CSIter == Map.end()) {
       LastUnmappedId = CSId;
@@ -886,8 +889,8 @@ struct LinearCallStackIdConverter {
                              std::function<Frame(LinearFrameId)> FrameIdToFrame)
       : CallStackBase(CallStackBase), FrameIdToFrame(FrameIdToFrame) {}
 
-  llvm::SmallVector<Frame> operator()(LinearCallStackId LinearCSId) {
-    llvm::SmallVector<Frame> Frames;
+  std::vector<Frame> operator()(LinearCallStackId LinearCSId) {
+    std::vector<Frame> Frames;
 
     const unsigned char *Ptr =
         CallStackBase +
@@ -897,9 +900,18 @@ struct LinearCallStackIdConverter {
     Frames.reserve(NumFrames);
     for (; NumFrames; --NumFrames) {
       LinearFrameId Elem =
-          support::endian::readNext<LinearFrameId, llvm::endianness::little>(
-              Ptr);
+          support::endian::read<LinearFrameId, llvm::endianness::little>(Ptr);
+      // Follow a pointer to the parent, if any.  See comments below on
+      // CallStackRadixTreeBuilder for the description of the radix tree format.
+      if (static_cast<std::make_signed_t<LinearFrameId>>(Elem) < 0) {
+        Ptr += (-Elem) * sizeof(LinearFrameId);
+        Elem =
+            support::endian::read<LinearFrameId, llvm::endianness::little>(Ptr);
+      }
+      // We shouldn't encounter another pointer.
+      assert(static_cast<std::make_signed_t<LinearFrameId>>(Elem) >= 0);
       Frames.push_back(FrameIdToFrame(Elem));
+      Ptr += sizeof(LinearFrameId);
     }
 
     return Frames;
@@ -920,6 +932,125 @@ struct IndexedMemProfData {
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> CallStackData;
 };
 
+struct FrameStat {
+  // The number of occurrences of a given FrameId.
+  uint64_t Count = 0;
+  // The sum of indexes where a given FrameId shows up.
+  uint64_t PositionSum = 0;
+};
+
+// Compute a histogram of Frames in call stacks.
+llvm::DenseMap<FrameId, FrameStat>
+computeFrameHistogram(llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
+                          &MemProfCallStackData);
+
+// Construct a radix tree of call stacks.
+//
+// A set of call stacks might look like:
+//
+// CallStackId 1:  f1 -> f2 -> f3
+// CallStackId 2:  f1 -> f2 -> f4 -> f5
+// CallStackId 3:  f1 -> f2 -> f4 -> f6
+// CallStackId 4:  f7 -> f8 -> f9
+//
+// where each fn refers to a stack frame.
+//
+// Since we expect a lot of common prefixes, we can compress the call stacks
+// into a radix tree like:
+//
+// CallStackId 1:  f1 -> f2 -> f3
+//                       |
+// CallStackId 2:        +---> f4 -> f5
+//                             |
+// CallStackId 3:              +---> f6
+//
+// CallStackId 4:  f7 -> f8 -> f9
+//
+// Now, we are interested in retrieving call stacks for a given CallStackId, so
+// we just need a pointer from a given call stack to its parent.  For example,
+// CallStackId 2 would point to CallStackId 1 as a parent.
+//
+// We serialize the radix tree above into a single array along with the length
+// of each call stack and pointers to the parent call stacks.
+//
+// Index:              0  1  2  3  4  5  6  7  8  9 10 11 12 13 14
+// Array:             L3 f9 f8 f7 L4 f6 J3 L4 f5 f4 J3 L3 f3 f2 f1
+//                     ^           ^        ^           ^
+//                     |           |        |           |
+// CallStackId 4:  0 --+           |        |           |
+// CallStackId 3:  4 --------------+        |           |
+// CallStackId 2:  7 -----------------------+           |
+// CallStackId 1: 11 -----------------------------------+
+//
+// - LN indicates the length of a call stack, encoded as ordinary integer N.
+//
+// - JN indicates a pointer to the parent, encoded as -N.
+//
+// The radix tree allows us to reconstruct call stacks in the leaf-to-root
+// order as we scan the array from left ro right while following pointers to
+// parents along the way
+//
+// For example, if we are decoding CallStackId 2, we start a forward traversal
+// at Index 7, noting the call stack length of 4 and obtaining f5 and f4.  When
+// we see J3 at Index 10, we resume a forward traversal at Index 13 = 10 + 3,
+// picking up f2 and f1.  We are done after collecting 4 frames as indicated at
+// the beginning of the traversal.
+//
+// On-disk IndexedMemProfRecord will refer to call stacks by their indexes into
+// the radix tree array, so we do not explicitly encode mappings like:
+// "CallStackId 1 -> 11".
+class CallStackRadixTreeBuilder {
+  // The radix tree array.
+  std::vector<LinearFrameId> RadixArray;
+
+  // Mapping from CallStackIds to indexes into RadixArray.
+  llvm::DenseMap<CallStackId, LinearCallStackId> CallStackPos;
+
+  // In build, we partition a given call stack into two parts -- the prefix
+  // that's common with the previously encoded call stack and the frames beyond
+  // the common prefix -- the unique portion.  Then we want to find out where
+  // the common prefix is stored in RadixArray so that we can link the unique
+  // portion to the common prefix.  Indexes, declared below, helps with our
+  // needs.  Intuitively, Indexes tells us where each of the previously encoded
+  // call stack is stored in RadixArray.  More formally, Indexes satisfies:
+  //
+  //   RadixArray[Indexes[I]] == Prev[I]
+  //
+  // for every I, where Prev is the the call stack in the root-to-leaf order
+  // previously encoded by build.  (Note that Prev, as passed to
+  // encodeCallStack, is in the leaf-to-root order.)
+  //
+  // For example, if the call stack being encoded shares 5 frames at the root of
+  // the call stack with the previously encoded call stack,
+  // RadixArray[Indexes[0]] is the root frame of the common prefix.
+  // RadixArray[Indexes[5 - 1]] is the last frame of the common prefix.
+  std::vector<LinearCallStackId> Indexes;
+
+  using CSIdPair = std::pair<CallStackId, llvm::SmallVector<FrameId>>;
+
+  // Encode a call stack into RadixArray.  Return the starting index within
+  // RadixArray.
+  LinearCallStackId encodeCallStack(
+      const llvm::SmallVector<FrameId> *CallStack,
+      const llvm::SmallVector<FrameId> *Prev,
+      const llvm::DenseMap<FrameId, LinearFrameId> &MemProfFrameIndexes);
+
+public:
+  CallStackRadixTreeBuilder() = default;
+
+  // Build a radix tree array.
+  void build(llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
+                 &&MemProfCallStackData,
+             const llvm::DenseMap<FrameId, LinearFrameId> &MemProfFrameIndexes,
+             llvm::DenseMap<FrameId, FrameStat> &FrameHistogram);
+
+  const std::vector<LinearFrameId> &getRadixArray() const { return RadixArray; }
+
+  llvm::DenseMap<CallStackId, LinearCallStackId> takeCallStackPos() {
+    return std::move(CallStackPos);
+  }
+};
+
 // Verify that each CallStackId is computed with hashCallStack.  This function
 // is intended to help transition from CallStack to CSId in
 // IndexedAllocationInfo.
diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h
index b42e4f5977740995cedc15675414dfef9ee3e631..fbba6483abe3e26ba7af7c2056b009b33ccc331d 100644
--- a/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -76,8 +76,8 @@ public:
       Callback =
           std::bind(&MemProfReader::idToFrame, this, std::placeholders::_1);
 
-    memprof::CallStackIdConverter<decltype(CSIdToCallStack)> CSIdConv(
-        CSIdToCallStack, Callback);
+    CallStackIdConverter<decltype(CSIdToCallStack)> CSIdConv(CSIdToCallStack,
+                                                             Callback);
 
     const IndexedMemProfRecord &IndexedRecord = Iter->second;
     GuidRecord = {
@@ -137,7 +137,7 @@ class RawMemProfReader final : public MemProfReader {
 public:
   RawMemProfReader(const RawMemProfReader &) = delete;
   RawMemProfReader &operator=(const RawMemProfReader &) = delete;
-  virtual ~RawMemProfReader() override = default;
+  ~RawMemProfReader() override = default;
 
   // Prints the contents of the profile in YAML format.
   void printYAML(raw_ostream &OS);
@@ -161,7 +161,7 @@ public:
   // Returns a list of build ids recorded in the segment information.
   static std::vector<std::string> peekBuildIds(MemoryBuffer *DataBuffer);
 
-  virtual Error
+  Error
   readNextRecord(GuidMemProfRecordPair &GuidRecord,
                  std::function<const Frame(const FrameId)> Callback) override;
 
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 963a4d4918e567be478f2d28ff0832a956db0a0b..5398a44f13ba361c80e6f0add673468d6c4128e8 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -169,7 +169,7 @@ public:
 
 protected:
   SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS)
-      : SampleProfileWriter(OS), Indent(0) {}
+      : SampleProfileWriter(OS) {}
 
   std::error_code writeHeader(const SampleProfileMap &ProfileMap) override {
     LineCount = 0;
@@ -180,7 +180,7 @@ private:
   /// Indent level to use when writing.
   ///
   /// This is used when printing inlined callees.
-  unsigned Indent;
+  unsigned Indent = 0;
 
   friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
   SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h
index 46f99811763a81711cee022b1b6fac38837ef242..0e42789ba932efbe32922d39cc5f83fb6541cd62 100644
--- a/llvm/include/llvm/Support/CodeGen.h
+++ b/llvm/include/llvm/Support/CodeGen.h
@@ -87,7 +87,7 @@ namespace llvm {
   };
 
   // Specify what functions should keep the frame pointer.
-  enum class FramePointerKind { None, NonLeaf, All };
+  enum class FramePointerKind { None, NonLeaf, All, Reserved };
 
   // Specify what type of zeroing callee-used registers.
   namespace ZeroCallUsedRegs {
diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 30e0852b972c5a48dc91aff2db75078b4fd98359..5831fe66a1f7b71f26aac179c4e2c904d0ad4255 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -72,7 +72,8 @@ template <typename value_type, endianness endian, std::size_t alignment>
 
 /// Read a value of a particular endianness from a buffer, and increment the
 /// buffer past that value.
-template <typename value_type, std::size_t alignment, typename CharT>
+template <typename value_type, std::size_t alignment = unaligned,
+          typename CharT>
 [[nodiscard]] inline value_type readNext(const CharT *&memory,
                                          endianness endian) {
   value_type ret = read<value_type, alignment>(memory, endian);
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 29b5f9159c68ec493e2e155c18b4f3eba52a450b..b9d398fee2b74e1b6c8f44d743cfe4a9c3511650 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -638,7 +638,7 @@ struct SemiNCAInfo {
         Bucket;
     SmallDenseSet<TreeNodePtr, 8> Visited;
     SmallVector<TreeNodePtr, 8> Affected;
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     SmallVector<TreeNodePtr, 8> VisitedUnaffected;
 #endif
   };
@@ -913,7 +913,7 @@ struct SemiNCAInfo {
     LLVM_DEBUG(dbgs() << "Deleting edge " << BlockNamePrinter(From) << " -> "
                       << BlockNamePrinter(To) << "\n");
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     // Ensure that the edge was in fact deleted from the CFG before informing
     // the DomTree about it.
     // The check is O(N), so run it only in debug configuration.
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index 1e0d0ee446fc41ca3f43b2a27be46e610ee30db2..d19022729ace32dd717524c2b0773738932d811f 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -208,7 +208,7 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopPreheader() const {
     return nullptr;
 
   // Make sure there is only one exit out of the preheader.
-  if (llvm::size(llvm::children<BlockT *>(Out)) != 1)
+  if (!llvm::hasSingleElement(llvm::children<BlockT *>(Out)))
     return nullptr; // Multiple exits from the block, must not be a preheader.
 
   // The predecessor has exactly one successor, so it is a preheader.
diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h
index ba4a5f01036ca6b13aae5262d118576f43da5768..c206bd77d58800c122095641480d54037699033f 100644
--- a/llvm/include/llvm/Support/KnownBits.h
+++ b/llvm/include/llvm/Support/KnownBits.h
@@ -48,7 +48,6 @@ public:
 
   /// Returns true if we know the value of all bits.
   bool isConstant() const {
-    assert(!hasConflict() && "KnownBits conflict!");
     return Zero.popcount() + One.popcount() == getBitWidth();
   }
 
@@ -74,16 +73,10 @@ public:
   }
 
   /// Returns true if value is all zero.
-  bool isZero() const {
-    assert(!hasConflict() && "KnownBits conflict!");
-    return Zero.isAllOnes();
-  }
+  bool isZero() const { return Zero.isAllOnes(); }
 
   /// Returns true if value is all one bits.
-  bool isAllOnes() const {
-    assert(!hasConflict() && "KnownBits conflict!");
-    return One.isAllOnes();
-  }
+  bool isAllOnes() const { return One.isAllOnes(); }
 
   /// Make all bits known to be zero and discard any previous information.
   void setAllZero() {
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 1ea2652871ab820fa0ff5e99df04d9941716a2ea..bd43b958990302840ef5fe3ab71d179df262142d 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1605,6 +1605,37 @@ def insert_vector_elt_oob : GICombineRule<
          [{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
 
+def add_of_vscale : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_VSCALE $left, $imm1),
+          (G_VSCALE $right, $imm2),
+          (G_ADD $root, $left, $right, (MIFlags NoSWrap)),
+   [{ return Helper.matchAddOfVScale(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
+def mul_of_vscale : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_VSCALE $left, $scale),
+          (G_CONSTANT $x, $imm1),
+          (G_MUL $root, $left, $x, (MIFlags NoSWrap)),
+   [{ return Helper.matchMulOfVScale(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
+def shl_of_vscale : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_VSCALE $left, $imm),
+          (G_CONSTANT $x, $imm1),
+          (G_SHL $root, $left, $x, (MIFlags NoSWrap)),
+   [{ return Helper.matchShlOfVScale(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
+def sub_of_vscale : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_VSCALE $right, $imm),
+          (G_SUB $root, $x, $right, (MIFlags NoSWrap)),
+   [{ return Helper.matchSubOfVScale(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
 // match_extract_of_element and insert_vector_elt_oob must be the first!
 def vector_ops_combines: GICombineGroup<[
 match_extract_of_element_undef_vector,
@@ -1637,7 +1668,11 @@ extract_vector_element_build_vector_trunc6,
 extract_vector_element_build_vector_trunc7,
 extract_vector_element_build_vector_trunc8,
 extract_vector_element_shuffle_vector,
-insert_vector_element_extract_vector_element
+insert_vector_element_extract_vector_element,
+add_of_vscale,
+mul_of_vscale,
+shl_of_vscale,
+sub_of_vscale,
 ]>;
 
 
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 8fa0e4b86d6dc9c19d0bf5b3b162c9693b64e9d5..560d3b434d07d529734c5884a40af98c7206df5b 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -148,6 +148,7 @@ def : GINodeEquiv<G_BUILD_VECTOR, build_vector>;
 def : GINodeEquiv<G_FCEIL, fceil>;
 def : GINodeEquiv<G_FCOS, fcos>;
 def : GINodeEquiv<G_FSIN, fsin>;
+def : GINodeEquiv<G_FTAN, ftan>;
 def : GINodeEquiv<G_FABS, fabs>;
 def : GINodeEquiv<G_FSQRT, fsqrt>;
 def : GINodeEquiv<G_FFLOOR, ffloor>;
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 98a8b7ba337cb66fea9488eabc90c87e2cb97455..d3464b5202ff3233c7585c0f9ca331fe13931fb8 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -161,6 +161,11 @@ namespace llvm {
     /// optimization should be disabled for the given machine function.
     bool DisableFramePointerElim(const MachineFunction &MF) const;
 
+    /// FramePointerIsReserved - This returns true if the frame pointer must
+    /// always either point to a new frame record or be un-modified in the given
+    /// function.
+    bool FramePointerIsReserved(const MachineFunction &MF) const;
+
     /// If greater than 0, override the default value of
     /// MCAsmInfo::BinutilsVersion.
     std::pair<int, int> BinutilsVersion{0, 0};
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 1c95a609098469b15321aec14472a542ec7895f4..15e02eb49271dff699634fbec0b28671f312e9f1 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -509,6 +509,7 @@ def fneg       : SDNode<"ISD::FNEG"       , SDTFPUnaryOp>;
 def fsqrt      : SDNode<"ISD::FSQRT"      , SDTFPUnaryOp>;
 def fsin       : SDNode<"ISD::FSIN"       , SDTFPUnaryOp>;
 def fcos       : SDNode<"ISD::FCOS"       , SDTFPUnaryOp>;
+def ftan       : SDNode<"ISD::FTAN"       , SDTFPUnaryOp>;
 def fexp2      : SDNode<"ISD::FEXP2"      , SDTFPUnaryOp>;
 def fexp10     : SDNode<"ISD::FEXP10"     , SDTFPUnaryOp>;
 def fpow       : SDNode<"ISD::FPOW"       , SDTFPBinOp>;
@@ -562,6 +563,8 @@ def strict_fsin       : SDNode<"ISD::STRICT_FSIN",
                                SDTFPUnaryOp, [SDNPHasChain]>;
 def strict_fcos       : SDNode<"ISD::STRICT_FCOS",
                                SDTFPUnaryOp, [SDNPHasChain]>;
+def strict_ftan       : SDNode<"ISD::STRICT_FTAN",
+                               SDTFPUnaryOp, [SDNPHasChain]>;
 def strict_fexp2      : SDNode<"ISD::STRICT_FEXP2",
                                SDTFPUnaryOp, [SDNPHasChain]>;
 def strict_fpow       : SDNode<"ISD::STRICT_FPOW",
@@ -1517,6 +1520,9 @@ def any_fsin       : PatFrags<(ops node:$src),
 def any_fcos       : PatFrags<(ops node:$src),
                               [(strict_fcos node:$src),
                                (fcos node:$src)]>;
+def any_ftan       : PatFrags<(ops node:$src),
+                              [(strict_ftan node:$src),
+                               (ftan node:$src)]>;
 def any_fexp2      : PatFrags<(ops node:$src),
                               [(strict_fexp2 node:$src),
                                (fexp2 node:$src)]>;
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 5025ab2491de8fdeea46a6dd242de5ce82b0f215..c6010d745b46edbd5aa9835733a7140c9de21feb 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -132,48 +132,6 @@ struct ExtensionInfo {
 #define EMIT_EXTENSIONS
 #include "llvm/TargetParser/AArch64TargetParserDef.inc"
 
-struct ExtensionSet {
-  // Set of extensions which are currently enabled.
-  ExtensionBitset Enabled;
-  // Set of extensions which have been enabled or disabled at any point. Used
-  // to avoid cluttering the cc1 command-line with lots of unneeded features.
-  ExtensionBitset Touched;
-  // Base architecture version, which we need to know because some feature
-  // dependencies change depending on this.
-  const ArchInfo *BaseArch;
-
-  ExtensionSet() : Enabled(), Touched(), BaseArch(nullptr) {}
-
-  // Enable the given architecture extension, and any other extensions it
-  // depends on. Does not change the base architecture, or follow dependencies
-  // between features which are only related by required arcitecture versions.
-  void enable(ArchExtKind E);
-
-  // Disable the given architecture extension, and any other extensions which
-  // depend on it. Does not change the base architecture, or follow
-  // dependencies between features which are only related by required
-  // arcitecture versions.
-  void disable(ArchExtKind E);
-
-  // Add default extensions for the given CPU. Records the base architecture,
-  // to later resolve dependencies which depend on it.
-  void addCPUDefaults(const CpuInfo &CPU);
-
-  // Add default extensions for the given architecture version. Records the
-  // base architecture, to later resolve dependencies which depend on it.
-  void addArchDefaults(const ArchInfo &Arch);
-
-  // Add or remove a feature based on a modifier string. The string must be of
-  // the form "<name>" to enable a feature or "no<name>" to disable it. This
-  // will also enable or disable any features as required by the dependencies
-  // between them.
-  bool parseModifier(StringRef Modifier);
-
-  // Convert the set of enabled extension to an LLVM feature list, appending
-  // them to Features.
-  void toLLVMFeatureList(std::vector<StringRef> &Features) const;
-};
-
 // Represents a dependency between two architecture extensions. Later is the
 // feature which was added to the architecture after Earlier, and expands the
 // functionality provided by it. If Later is enabled, then Earlier will also be
@@ -577,6 +535,70 @@ inline constexpr CpuInfo CpuInfos[] = {
                                AArch64::AEK_SHA2, AArch64::AEK_AES,
                                AArch64::AEK_MTE, AArch64::AEK_SB,
                                AArch64::AEK_SSBS, AArch64::AEK_CSSC})},
+    {"oryon-1", ARMV8_6A,
+     (AArch64::ExtensionBitset({AArch64::AEK_AES, AArch64::AEK_CRYPTO,
+                                AArch64::AEK_RAND, AArch64::AEK_SM4,
+                                AArch64::AEK_SHA3, AArch64::AEK_SHA2,
+                                AArch64::AEK_PROFILE}))},
+};
+
+struct ExtensionSet {
+  // Set of extensions which are currently enabled.
+  ExtensionBitset Enabled;
+  // Set of extensions which have been enabled or disabled at any point. Used
+  // to avoid cluttering the cc1 command-line with lots of unneeded features.
+  ExtensionBitset Touched;
+  // Base architecture version, which we need to know because some feature
+  // dependencies change depending on this.
+  const ArchInfo *BaseArch;
+
+  ExtensionSet() : Enabled(), Touched(), BaseArch(nullptr) {}
+
+  // Enable the given architecture extension, and any other extensions it
+  // depends on. Does not change the base architecture, or follow dependencies
+  // between features which are only related by required arcitecture versions.
+  void enable(ArchExtKind E);
+
+  // Disable the given architecture extension, and any other extensions which
+  // depend on it. Does not change the base architecture, or follow
+  // dependencies between features which are only related by required
+  // arcitecture versions.
+  void disable(ArchExtKind E);
+
+  // Add default extensions for the given CPU. Records the base architecture,
+  // to later resolve dependencies which depend on it.
+  void addCPUDefaults(const CpuInfo &CPU);
+
+  // Add default extensions for the given architecture version. Records the
+  // base architecture, to later resolve dependencies which depend on it.
+  void addArchDefaults(const ArchInfo &Arch);
+
+  // Add or remove a feature based on a modifier string. The string must be of
+  // the form "<name>" to enable a feature or "no<name>" to disable it. This
+  // will also enable or disable any features as required by the dependencies
+  // between them.
+  bool parseModifier(StringRef Modifier, const bool AllowNoDashForm = false);
+
+  // Constructs a new ExtensionSet by toggling the corresponding bits for every
+  // feature in the \p Features list without expanding their dependencies. Used
+  // for reconstructing an ExtensionSet from the output of toLLVMFeatures().
+  void reconstructFromParsedFeatures(const std::vector<std::string> &Features);
+
+  // Convert the set of enabled extension to an LLVM feature list, appending
+  // them to Features.
+  template <typename T> void toLLVMFeatureList(std::vector<T> &Features) const {
+    if (BaseArch && !BaseArch->ArchFeature.empty())
+      Features.emplace_back(T(BaseArch->ArchFeature));
+
+    for (const auto &E : Extensions) {
+      if (E.Feature.empty() || !Touched.test(E.ID))
+        continue;
+      if (Enabled.test(E.ID))
+        Features.emplace_back(T(E.Feature));
+      else
+        Features.emplace_back(T(E.NegFeature));
+    }
+  }
 };
 
 // Name alias.
@@ -602,7 +624,13 @@ const ArchInfo *getArchForCpu(StringRef CPU);
 
 // Parser
 const ArchInfo *parseArch(StringRef Arch);
+
+// Return the extension which has the given -target-feature name.
+std::optional<ExtensionInfo> targetFeatureToExtension(StringRef TargetFeature);
+
+// Parse a name as defined by the Extension class in tablegen.
 std::optional<ExtensionInfo> parseArchExtension(StringRef Extension);
+
 // Given the name of a CPU or alias, return the correponding CpuInfo.
 std::optional<CpuInfo> parseCpu(StringRef Name);
 // Used by target parser tests
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index d7b77a6ef5b68f7cfa4dec541e6bbdf88474b7d0..e5a1ce54fd46a74555d64d8bb4f17e9895aa1eaf 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -330,6 +330,7 @@ ARM_CPU_NAME("cortex-r7", ARMV7R, FK_VFPV3_D16_FP16, false,
 ARM_CPU_NAME("cortex-r8", ARMV7R, FK_VFPV3_D16_FP16, false,
              (ARM::AEK_MP | ARM::AEK_HWDIVARM))
 ARM_CPU_NAME("cortex-r52", ARMV8R, FK_NEON_FP_ARMV8, false, ARM::AEK_NONE)
+ARM_CPU_NAME("cortex-r52plus", ARMV8R, FK_NEON_FP_ARMV8, false, ARM::AEK_NONE)
 ARM_CPU_NAME("sc300", ARMV7M, FK_NONE, false, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m3", ARMV7M, FK_NONE, true, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE)
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index 8fc6fa37cb9aa7cc2959643747d2f7c8a7222d5f..e03d8f6eebfca36a96b7f06e09ace7fb1a0154d2 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -105,6 +105,7 @@ enum GPUKind : uint32_t {
   GK_GFX1103 = 93,
   GK_GFX1150 = 94,
   GK_GFX1151 = 95,
+  GK_GFX1152 = 96,
 
   GK_GFX1200 = 100,
   GK_GFX1201 = 101,
diff --git a/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h b/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h
new file mode 100644
index 0000000000000000000000000000000000000000..4c5a1b61e2d4473c8e36f46b99273707969db800
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h
@@ -0,0 +1,40 @@
+//===- ExpandVariadics.h - expand variadic functions ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H
+#define LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Module;
+class ModulePass;
+class OptimizationLevel;
+
+enum class ExpandVariadicsMode {
+  Unspecified, // Use the implementation defaults
+  Disable,     // Disable the pass entirely
+  Optimize,    // Optimise without changing ABI
+  Lowering,    // Change variadic calling convention
+};
+
+class ExpandVariadicsPass : public PassInfoMixin<ExpandVariadicsPass> {
+  const ExpandVariadicsMode Mode;
+
+public:
+  // Operates under passed mode unless overridden on commandline
+  ExpandVariadicsPass(ExpandVariadicsMode Mode);
+
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+ModulePass *createExpandVariadicsPass(ExpandVariadicsMode);
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H
diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index 65d43775bdc1d1c8774cc452e54132316f1fea44..290e5a1cc337f36cb42f1c26c5d8261b5d5abf46 100644
--- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -88,7 +88,7 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
   std::optional<BranchProbabilityInfo *> BPI;
   bool ChangedSinceLastAnalysisUpdate = false;
   bool HasGuards = false;
-#ifndef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if !LLVM_ENABLE_ABI_BREAKING_CHECKS
   SmallPtrSet<const BasicBlock *, 16> LoopHeaders;
 #else
   SmallSet<AssertingVH<const BasicBlock>, 16> LoopHeaders;
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 6aab1f98e6781648107e3363fced952b70b7daee..227f6dfb8362cb61ccbba93e746cbdba4fd2cc3f 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -286,7 +286,7 @@ public:
   }
 
   void setParentLoop(Loop *L) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     ParentL = L;
 #endif
   }
@@ -377,7 +377,7 @@ private:
   const bool LoopNestMode;
   bool LoopNestChanged;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   // In debug builds we also track the parent loop to implement asserts even in
   // the face of loop deletion.
   Loop *ParentL;
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 62c1e15a9a60e17c6b0f831f6f9dc32da8b3a525..468b50092efcf770f0bb953522b66b547e85be9f 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -167,7 +167,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   /// consistent when instructions are moved.
   SmallVector<SCEVInsertPointGuard *, 8> InsertPointGuards;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   const char *DebugType;
 #endif
 
@@ -183,7 +183,7 @@ public:
         Builder(se.getContext(), InstSimplifyFolder(DL),
                 IRBuilderCallbackInserter(
                     [this](Instruction *I) { rememberInstruction(I); })) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     DebugType = "";
 #endif
   }
@@ -193,7 +193,7 @@ public:
     assert(InsertPointGuards.empty());
   }
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   void setDebugType(const char *s) { DebugType = s; }
 #endif
 
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index bd804dc1126624eb2d41b9750889d67cd84ecb6c..797c082333a76c82134be7535d1c8d74a30617bd 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_UTILS_UNROLLLOOP_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/Analysis/CodeMetrics.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Support/InstructionCost.h"
 
@@ -73,6 +74,7 @@ struct UnrollLoopOptions {
   bool AllowExpensiveTripCount;
   bool UnrollRemainder;
   bool ForgetAllSCEV;
+  const Instruction *Heart = nullptr;
 };
 
 LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -128,14 +130,15 @@ class UnrollCostEstimator {
 
 public:
   unsigned NumInlineCandidates;
-  bool Convergent;
+  ConvergenceKind Convergence;
+  bool ConvergenceAllowsRuntime;
 
   UnrollCostEstimator(const Loop *L, const TargetTransformInfo &TTI,
                       const SmallPtrSetImpl<const Value *> &EphValues,
                       unsigned BEInsns);
 
   /// Whether it is legal to unroll this loop.
-  bool canUnroll() const { return LoopSize.isValid() && !NotDuplicatable; }
+  bool canUnroll() const;
 
   uint64_t getRolledLoopSize() const { return *LoopSize.getValue(); }
 
diff --git a/llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.h b/llvm/include/llvm/Transforms/Vectorize/LoopIdiomVectorize.h
similarity index 60%
rename from llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.h
rename to llvm/include/llvm/Transforms/Vectorize/LoopIdiomVectorize.h
index cc68425bb68b581483f66848a0c102ad6573df92..56f44b7dc6b2acd15a38084fca68baa3f523666a 100644
--- a/llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopIdiomVectorize.h
@@ -1,4 +1,4 @@
-//===- AArch64LoopIdiomTransform.h --------------------------------------===//
+//===----------LoopIdiomVectorize.h -----------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,20 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64LOOPIDIOMTRANSFORM_H
-#define LLVM_LIB_TARGET_AARCH64_AARCH64LOOPIDIOMTRANSFORM_H
+#ifndef LLVM_LIB_TRANSFORMS_VECTORIZE_LOOPIDIOMVECTORIZE_H
+#define LLVM_LIB_TRANSFORMS_VECTORIZE_LOOPIDIOMVECTORIZE_H
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 
 namespace llvm {
-
-struct AArch64LoopIdiomTransformPass
-    : PassInfoMixin<AArch64LoopIdiomTransformPass> {
+struct LoopIdiomVectorizePass : PassInfoMixin<LoopIdiomVectorizePass> {
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
 };
-
 } // namespace llvm
-
-#endif // LLVM_LIB_TARGET_AARCH64_AARCH64LOOPIDIOMTRANSFORM_H
+#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_LOOPIDIOMVECTORIZE_H
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 2246887afe68a6d67262931064ff13eabfa98e88..8ae5c3dee6103e9f28630de4caeff27356ace3cb 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -340,6 +340,11 @@ ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
     } while (!RCWorklist.empty());
   }
 
+#if defined(EXPENSIVE_CHECKS)
+  // Verify that the call graph is still valid.
+  CG.verify();
+#endif
+
   // By definition we preserve the call garph, all SCC analyses, and the
   // analysis proxies by handling them above and in any nested pass managers.
   PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
diff --git a/llvm/lib/Analysis/CodeMetrics.cpp b/llvm/lib/Analysis/CodeMetrics.cpp
index 2637e2f97dbb21ba3932f73382845326cfd41eb6..ea67b526423bf5bf33e10083537204b9a06e8f12 100644
--- a/llvm/lib/Analysis/CodeMetrics.cpp
+++ b/llvm/lib/Analysis/CodeMetrics.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InstructionCost.h"
 
@@ -111,11 +112,24 @@ void CodeMetrics::collectEphemeralValues(
   completeEphemeralValues(Visited, Worklist, EphValues);
 }
 
+static bool extendsConvergenceOutsideLoop(const Instruction &I, const Loop *L) {
+  if (!L)
+    return false;
+  if (!isa<ConvergenceControlInst>(I))
+    return false;
+  for (const auto *U : I.users()) {
+    if (!L->contains(cast<Instruction>(U)))
+      return true;
+  }
+  return false;
+}
+
 /// Fill in the current structure with information gleaned from the specified
 /// block.
 void CodeMetrics::analyzeBasicBlock(
     const BasicBlock *BB, const TargetTransformInfo &TTI,
-    const SmallPtrSetImpl<const Value *> &EphValues, bool PrepareForLTO) {
+    const SmallPtrSetImpl<const Value *> &EphValues, bool PrepareForLTO,
+    const Loop *L) {
   ++NumBlocks;
   InstructionCost NumInstsBeforeThisBB = NumInsts;
   for (const Instruction &I : *BB) {
@@ -163,19 +177,38 @@ void CodeMetrics::analyzeBasicBlock(
     if (isa<ExtractElementInst>(I) || I.getType()->isVectorTy())
       ++NumVectorInsts;
 
-    if (I.getType()->isTokenTy() && I.isUsedOutsideOfBlock(BB))
+    if (I.getType()->isTokenTy() && !isa<ConvergenceControlInst>(I) &&
+        I.isUsedOutsideOfBlock(BB)) {
+      LLVM_DEBUG(dbgs() << I
+                        << "\n  Cannot duplicate a token value used outside "
+                           "the current block (except convergence control).\n");
       notDuplicatable = true;
-
-    if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
-      if (CI->cannotDuplicate())
-        notDuplicatable = true;
-      if (CI->isConvergent())
-        convergent = true;
     }
 
-    if (const InvokeInst *InvI = dyn_cast<InvokeInst>(&I))
-      if (InvI->cannotDuplicate())
+    if (const CallBase *CB = dyn_cast<CallBase>(&I)) {
+      if (CB->cannotDuplicate())
         notDuplicatable = true;
+      // Compute a meet over the visited blocks for the following partial order:
+      //
+      // None -> { Controlled, ExtendedLoop, Uncontrolled}
+      // Controlled -> ExtendedLoop
+      if (Convergence <= ConvergenceKind::Controlled && CB->isConvergent()) {
+        if (isa<ConvergenceControlInst>(CB) ||
+            CB->getConvergenceControlToken()) {
+          assert(Convergence != ConvergenceKind::Uncontrolled);
+          LLVM_DEBUG(dbgs() << "Found controlled convergence:\n" << I << "\n");
+          if (extendsConvergenceOutsideLoop(I, L))
+            Convergence = ConvergenceKind::ExtendedLoop;
+          else {
+            assert(Convergence != ConvergenceKind::ExtendedLoop);
+            Convergence = ConvergenceKind::Controlled;
+          }
+        } else {
+          assert(Convergence == ConvergenceKind::None);
+          Convergence = ConvergenceKind::Uncontrolled;
+        }
+      }
+    }
 
     NumInsts += TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize);
   }
diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp
index 1e5cf84d589b02c50d2b98c913fbd0106986a4cc..48a7ca0061600b9f78d09c28fc7bbb4362f95f51 100644
--- a/llvm/lib/Analysis/LazyCallGraph.cpp
+++ b/llvm/lib/Analysis/LazyCallGraph.cpp
@@ -211,6 +211,14 @@ LazyCallGraph::LazyCallGraph(LazyCallGraph &&G)
   updateGraphPtrs();
 }
 
+#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
+void LazyCallGraph::verify() {
+  for (RefSCC &RC : postorder_ref_sccs()) {
+    RC.verify();
+  }
+}
+#endif
+
 bool LazyCallGraph::invalidate(Module &, const PreservedAnalyses &PA,
                                ModuleAnalysisManager::Invalidator &) {
   // Check whether the analysis, all analyses on functions, or the function's
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 6cded828c25f4a6f9e61bf3cecedf13971c17507..8b6e56ce8113dac3588b494fbb6ca63f6a700689 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1116,9 +1116,6 @@ LazyValueInfoImpl::getValueFromSimpleICmpCondition(CmpInst::Predicate Pred,
     if (!R)
       return std::nullopt;
     RHSRange = toConstantRange(*R, RHS->getType());
-  } else if (Instruction *I = dyn_cast<Instruction>(RHS)) {
-    if (auto *Ranges = I->getMetadata(LLVMContext::MD_range))
-      RHSRange = getConstantRangeFromMetadata(*Ranges);
   }
 
   ConstantRange TrueValues =
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 369ab087ffc0f8826f8c103d63f44fecc2b7ae60..c34c4974382ea6348bc6beb04f5ff78db9d9c3ea 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -1105,6 +1105,26 @@ int llvm::getIntLoopAttribute(const Loop *TheLoop, StringRef Name,
   return getOptionalIntLoopAttribute(TheLoop, Name).value_or(Default);
 }
 
+CallBase *llvm::getLoopConvergenceHeart(const Loop *TheLoop) {
+  BasicBlock *H = TheLoop->getHeader();
+  for (Instruction &II : *H) {
+    if (auto *CB = dyn_cast<CallBase>(&II)) {
+      if (!CB->isConvergent())
+        continue;
+      // This is the heart if it uses a token defined outside the loop. The
+      // verifier has already checked that only the loop intrinsic can use such
+      // a token.
+      if (auto *Token = CB->getConvergenceControlToken()) {
+        auto *TokenDef = cast<Instruction>(Token);
+        if (!TheLoop->contains(TokenDef->getParent()))
+          return CB;
+      }
+      return nullptr;
+    }
+  }
+  return nullptr;
+}
+
 bool llvm::isFinite(const Loop *L) {
   return L->getHeader()->getParent()->willReturn();
 }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 08138a5e2f2d9d73f152deb672e9bf50eb00b5fd..6a806d596efe75fd7c0c19a7adc57493b9c4ed24 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1158,7 +1158,6 @@ static void computeKnownBitsFromOperator(const Operator *I,
           Known.makeNonNegative();
       }
 
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       break;
     }
 
@@ -2055,8 +2054,6 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
 
   // Check whether we can determine known bits from context such as assumes.
   computeKnownBitsFromContext(V, Known, Depth, Q);
-
-  assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
 }
 
 /// Try to detect a recurrence that the value of the induction variable is
@@ -7296,10 +7293,13 @@ static bool isGuaranteedNotToBeUndefOrPoison(
         isa<ConstantPointerNull>(C) || isa<Function>(C))
       return true;
 
-    if (C->getType()->isVectorTy() && !isa<ConstantExpr>(C))
-      return (!includesUndef(Kind) ? !C->containsPoisonElement()
-                                   : !C->containsUndefOrPoisonElement()) &&
-             !C->containsConstantExpression();
+    if (C->getType()->isVectorTy() && !isa<ConstantExpr>(C)) {
+      if (includesUndef(Kind) && C->containsUndefElement())
+        return false;
+      if (includesPoison(Kind) && C->containsPoisonElement())
+        return false;
+      return !C->containsConstantExpression();
+    }
   }
 
   // Strip cast operations from a pointer value.
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 917094267d05ae425a2ca837cc1a464af51e6351..30728ed5875090599efe898907b868a6ab857c7b 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -68,6 +68,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   case Intrinsic::sqrt: // Begin floating-point.
   case Intrinsic::sin:
   case Intrinsic::cos:
+  case Intrinsic::tan:
   case Intrinsic::exp:
   case Intrinsic::exp2:
   case Intrinsic::log:
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index d3ab306904da123e9a3ec45d9629fc8ea2d1f125..7d7fe19568e8a6582b9474f2f9de649749379239 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -604,6 +604,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(aarch64_vector_pcs);
   KEYWORD(aarch64_sve_vector_pcs);
   KEYWORD(aarch64_sme_preservemost_from_x0);
+  KEYWORD(aarch64_sme_preservemost_from_x1);
   KEYWORD(aarch64_sme_preservemost_from_x2);
   KEYWORD(msp430_intrcc);
   KEYWORD(avr_intrcc);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 07c8aa23fc5e21beb68efc3818489f0d7dd6f7c8..eb1e3e494a42f4d6eb7f6b582eb8667f21fd3154 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -74,23 +74,6 @@ static std::string getTypeString(Type *T) {
   return Tmp.str();
 }
 
-// Whatever debug info format we parsed, we should convert to the expected debug
-// info format immediately afterwards.
-bool LLParser::finalizeDebugInfoFormat(Module *M) {
-  // We should have already returned an error if we observed both intrinsics and
-  // records in this IR.
-  assert(!(SeenNewDbgInfoFormat && SeenOldDbgInfoFormat) &&
-         "Mixed debug intrinsics/records seen without a parsing error?");
-  if (PreserveInputDbgFormat == cl::boolOrDefault::BOU_TRUE) {
-    UseNewDbgInfoFormat = SeenNewDbgInfoFormat;
-    WriteNewDbgInfoFormatToBitcode = SeenNewDbgInfoFormat;
-    WriteNewDbgInfoFormat = SeenNewDbgInfoFormat;
-  } else if (M) {
-    M->setIsNewDbgInfoFormat(false);
-  }
-  return false;
-}
-
 /// Run: module ::= toplevelentity*
 bool LLParser::Run(bool UpgradeDebugInfo,
                    DataLayoutCallbackTy DataLayoutCallback) {
@@ -108,7 +91,7 @@ bool LLParser::Run(bool UpgradeDebugInfo,
   }
 
   return parseTopLevelEntities() || validateEndOfModule(UpgradeDebugInfo) ||
-         validateEndOfIndex() || finalizeDebugInfoFormat(M);
+         validateEndOfIndex();
 }
 
 bool LLParser::parseStandaloneConstantValue(Constant *&C,
@@ -207,6 +190,18 @@ void LLParser::dropUnknownMetadataReferences() {
 bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
   if (!M)
     return false;
+
+  // We should have already returned an error if we observed both intrinsics and
+  // records in this IR.
+  assert(!(SeenNewDbgInfoFormat && SeenOldDbgInfoFormat) &&
+         "Mixed debug intrinsics/records seen without a parsing error?");
+  if (PreserveInputDbgFormat == cl::boolOrDefault::BOU_TRUE) {
+    UseNewDbgInfoFormat = SeenNewDbgInfoFormat;
+    WriteNewDbgInfoFormatToBitcode = SeenNewDbgInfoFormat;
+    WriteNewDbgInfoFormat = SeenNewDbgInfoFormat;
+    M->setNewDbgInfoFormatFlag(SeenNewDbgInfoFormat);
+  }
+
   // Handle any function attribute group forward references.
   for (const auto &RAG : ForwardRefAttrGroups) {
     Value *V = RAG.first;
@@ -439,6 +434,9 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
   UpgradeModuleFlags(*M);
   UpgradeSectionAttributes(*M);
 
+  if (PreserveInputDbgFormat != cl::boolOrDefault::BOU_TRUE)
+    M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
+
   if (!Slots)
     return false;
   // Initialize the slot mapping.
@@ -2153,6 +2151,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) {
 ///   ::= 'aarch64_vector_pcs'
 ///   ::= 'aarch64_sve_vector_pcs'
 ///   ::= 'aarch64_sme_preservemost_from_x0'
+///   ::= 'aarch64_sme_preservemost_from_x1'
 ///   ::= 'aarch64_sme_preservemost_from_x2'
 ///   ::= 'msp430_intrcc'
 ///   ::= 'avr_intrcc'
@@ -2212,6 +2211,9 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) {
   case lltok::kw_aarch64_sme_preservemost_from_x0:
     CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0;
     break;
+  case lltok::kw_aarch64_sme_preservemost_from_x1:
+    CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1;
+    break;
   case lltok::kw_aarch64_sme_preservemost_from_x2:
     CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2;
     break;
diff --git a/llvm/lib/BinaryFormat/Dwarf.cpp b/llvm/lib/BinaryFormat/Dwarf.cpp
index 73242661726840084f8a209abab2d6cde0bd146f..0cd5dfbd023e4e91b88d5f7c722e2927e45b4e6e 100644
--- a/llvm/lib/BinaryFormat/Dwarf.cpp
+++ b/llvm/lib/BinaryFormat/Dwarf.cpp
@@ -139,7 +139,7 @@ StringRef llvm::dwarf::OperationEncodingString(unsigned Encoding) {
   switch (Encoding) {
   default:
     return StringRef();
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)                                \
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
   case DW_OP_##NAME:                                                           \
     return "DW_OP_" #NAME;
 #include "llvm/BinaryFormat/Dwarf.def"
@@ -155,12 +155,16 @@ StringRef llvm::dwarf::OperationEncodingString(unsigned Encoding) {
     return "DW_OP_LLVM_implicit_pointer";
   case DW_OP_LLVM_arg:
     return "DW_OP_LLVM_arg";
+  case DW_OP_LLVM_extract_bits_sext:
+    return "DW_OP_LLVM_extract_bits_sext";
+  case DW_OP_LLVM_extract_bits_zext:
+    return "DW_OP_LLVM_extract_bits_zext";
   }
 }
 
 unsigned llvm::dwarf::getOperationEncoding(StringRef OperationEncodingString) {
   return StringSwitch<unsigned>(OperationEncodingString)
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)                                \
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
   .Case("DW_OP_" #NAME, DW_OP_##NAME)
 #include "llvm/BinaryFormat/Dwarf.def"
       .Case("DW_OP_LLVM_convert", DW_OP_LLVM_convert)
@@ -169,6 +173,8 @@ unsigned llvm::dwarf::getOperationEncoding(StringRef OperationEncodingString) {
       .Case("DW_OP_LLVM_entry_value", DW_OP_LLVM_entry_value)
       .Case("DW_OP_LLVM_implicit_pointer", DW_OP_LLVM_implicit_pointer)
       .Case("DW_OP_LLVM_arg", DW_OP_LLVM_arg)
+      .Case("DW_OP_LLVM_extract_bits_sext", DW_OP_LLVM_extract_bits_sext)
+      .Case("DW_OP_LLVM_extract_bits_zext", DW_OP_LLVM_extract_bits_zext)
       .Default(0);
 }
 
@@ -210,18 +216,44 @@ unsigned llvm::dwarf::OperationVersion(dwarf::LocationAtom Op) {
   switch (Op) {
   default:
     return 0;
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)                                \
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
   case DW_OP_##NAME:                                                           \
     return VERSION;
 #include "llvm/BinaryFormat/Dwarf.def"
   }
 }
 
+std::optional<unsigned> llvm::dwarf::OperationOperands(dwarf::LocationAtom Op) {
+  switch (Op) {
+  default:
+    return std::nullopt;
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
+  case DW_OP_##NAME:                                                           \
+    if (OPERANDS == -1)                                                        \
+      return std::nullopt;                                                     \
+    return OPERANDS;
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+}
+
+std::optional<unsigned> llvm::dwarf::OperationArity(dwarf::LocationAtom Op) {
+  switch (Op) {
+  default:
+    return std::nullopt;
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
+  case DW_OP_##NAME:                                                           \
+    if (ARITY == -1)                                                           \
+      return std::nullopt;                                                     \
+    return ARITY;
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+}
+
 unsigned llvm::dwarf::OperationVendor(dwarf::LocationAtom Op) {
   switch (Op) {
   default:
     return 0;
-#define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)                                \
+#define HANDLE_DW_OP(ID, NAME, OPERANDS, ARITY, VERSION, VENDOR)               \
   case DW_OP_##NAME:                                                           \
     return DWARF_VENDOR_##VENDOR;
 #include "llvm/BinaryFormat/Dwarf.def"
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 8519796deeb1a87598fec53744cf8f89a36037b6..026595bdc6340538fdbea2c4d12ad5bdf737ab3a 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -4355,7 +4355,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
   if (PreserveInputDbgFormat != cl::boolOrDefault::BOU_TRUE) {
     TheModule->IsNewDbgInfoFormat =
         UseNewDbgInfoFormat &&
-        LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_TRUE;
+        LoadBitcodeIntoNewDbgInfoFormat != cl::boolOrDefault::BOU_FALSE;
   }
 
   this->ValueTypeCallback = std::move(Callbacks.ValueType);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index a74d43897d45b30a43fc3db85859f21a9966e201..cc96d3c481f70147cf7ca15b3d5f9f98dab378c6 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -18,6 +18,7 @@
 #include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
 
@@ -546,6 +547,37 @@ bool DwarfExpression::addExpression(
       LocationKind = Unknown;
       return true;
     }
+    case dwarf::DW_OP_LLVM_extract_bits_sext:
+    case dwarf::DW_OP_LLVM_extract_bits_zext: {
+      unsigned SizeInBits = Op->getArg(1);
+      unsigned BitOffset = Op->getArg(0);
+
+      // If we have a memory location then dereference to get the value
+      if (isMemoryLocation())
+        emitOp(dwarf::DW_OP_deref);
+
+      // Extract the bits by a shift left (to shift out the bits after what we
+      // want to extract) followed by shift right (to shift the bits to position
+      // 0 and also sign/zero extend). These operations are done in the DWARF
+      // "generic type" whose size is the size of a pointer.
+      unsigned PtrSizeInBytes = CU.getAsmPrinter()->MAI->getCodePointerSize();
+      unsigned LeftShift = PtrSizeInBytes * 8 - (SizeInBits + BitOffset);
+      unsigned RightShift = LeftShift + BitOffset;
+      if (LeftShift) {
+        emitOp(dwarf::DW_OP_constu);
+        emitUnsigned(LeftShift);
+        emitOp(dwarf::DW_OP_shl);
+      }
+      emitOp(dwarf::DW_OP_constu);
+      emitUnsigned(RightShift);
+      emitOp(OpNum == dwarf::DW_OP_LLVM_extract_bits_sext ? dwarf::DW_OP_shra
+                                                          : dwarf::DW_OP_shr);
+
+      // The value is now at the top of the stack, so set the location to
+      // implicit so that we get a stack_value at the end.
+      LocationKind = Implicit;
+      break;
+    }
     case dwarf::DW_OP_plus_uconst:
       assert(!isRegisterLocation());
       emitOp(dwarf::DW_OP_plus_uconst);
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 677460a2d8e40178e40bbf3b76e08ed983a7fde0..8fc65d78ff2c9068c55f27babef2158fb4063dd0 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -211,6 +211,9 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
                      "Disable frame pointer elimination"),
           clEnumValN(FramePointerKind::NonLeaf, "non-leaf",
                      "Disable frame pointer elimination for non-leaf frame"),
+          clEnumValN(FramePointerKind::Reserved, "reserved",
+                     "Enable frame pointer elimination, but reserve the frame "
+                     "pointer register"),
           clEnumValN(FramePointerKind::None, "none",
                      "Enable frame pointer elimination")));
   CGBINDOPT(FramePointerUsage);
@@ -693,6 +696,8 @@ void codegen::setFunctionAttributes(StringRef CPU, StringRef Features,
       NewAttrs.addAttribute("frame-pointer", "all");
     else if (getFramePointerUsage() == FramePointerKind::NonLeaf)
       NewAttrs.addAttribute("frame-pointer", "non-leaf");
+    else if (getFramePointerUsage() == FramePointerKind::Reserved)
+      NewAttrs.addAttribute("frame-pointer", "reserved");
     else if (getFramePointerUsage() == FramePointerKind::None)
       NewAttrs.addAttribute("frame-pointer", "none");
   }
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
index b4765fb280f9dd489a1c20f66ce1cb8612116810..66b1c5f8ca82c7416bb75443d3d61613de433023 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT.
+// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT,
+// G_INSERT_VECTOR_ELT, and G_VSCALE
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
@@ -400,3 +401,86 @@ bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
 
   return false;
 }
+
+bool CombinerHelper::matchAddOfVScale(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GAdd *Add = cast<GAdd>(MRI.getVRegDef(MO.getReg()));
+  GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Add->getLHSReg()));
+  GVScale *RHSVScale = cast<GVScale>(MRI.getVRegDef(Add->getRHSReg()));
+
+  Register Dst = Add->getReg(0);
+
+  if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) ||
+      !MRI.hasOneNonDBGUse(RHSVScale->getReg(0)))
+    return false;
+
+  MatchInfo = [=](MachineIRBuilder &B) {
+    B.buildVScale(Dst, LHSVScale->getSrc() + RHSVScale->getSrc());
+  };
+
+  return true;
+}
+
+bool CombinerHelper::matchMulOfVScale(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GMul *Mul = cast<GMul>(MRI.getVRegDef(MO.getReg()));
+  GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Mul->getLHSReg()));
+
+  std::optional<APInt> MaybeRHS = getIConstantVRegVal(Mul->getRHSReg(), MRI);
+  if (!MaybeRHS)
+    return false;
+
+  Register Dst = MO.getReg();
+
+  if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)))
+    return false;
+
+  MatchInfo = [=](MachineIRBuilder &B) {
+    B.buildVScale(Dst, LHSVScale->getSrc() * *MaybeRHS);
+  };
+
+  return true;
+}
+
+bool CombinerHelper::matchSubOfVScale(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GSub *Sub = cast<GSub>(MRI.getVRegDef(MO.getReg()));
+  GVScale *RHSVScale = cast<GVScale>(MRI.getVRegDef(Sub->getRHSReg()));
+
+  Register Dst = MO.getReg();
+  LLT DstTy = MRI.getType(Dst);
+
+  if (!MRI.hasOneNonDBGUse(RHSVScale->getReg(0)) ||
+      !isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, DstTy}))
+    return false;
+
+  MatchInfo = [=](MachineIRBuilder &B) {
+    auto VScale = B.buildVScale(DstTy, -RHSVScale->getSrc());
+    B.buildAdd(Dst, Sub->getLHSReg(), VScale, Sub->getFlags());
+  };
+
+  return true;
+}
+
+bool CombinerHelper::matchShlOfVScale(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GShl *Shl = cast<GShl>(MRI.getVRegDef(MO.getReg()));
+  GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Shl->getSrcReg()));
+
+  std::optional<APInt> MaybeRHS = getIConstantVRegVal(Shl->getShiftReg(), MRI);
+  if (!MaybeRHS)
+    return false;
+
+  Register Dst = MO.getReg();
+  LLT DstTy = MRI.getType(Dst);
+
+  if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) ||
+      !isLegalOrBeforeLegalizer({TargetOpcode::G_VSCALE, DstTy}))
+    return false;
+
+  MatchInfo = [=](MachineIRBuilder &B) {
+    B.buildVScale(Dst, LHSVScale->getSrc().shl(*MaybeRHS));
+  };
+
+  return true;
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 32d607cfd71a7e70615b7545f4b9bcea9f855233..2988fa45b2e336a170fe3a115b67dc749e0a8497 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -607,7 +607,6 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
   }
   }
 
-  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
   LLVM_DEBUG(dumpResult(MI, Known, Depth));
 
   // Update the cache.
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 6f0cae2edab17f38b2042bd0e92bd11ad2f670c0..9830b521797c1b54b12c1c35c377cdeeedb42d6f 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -449,6 +449,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     RTLIBCASE(SIN_F);
   case TargetOpcode::G_FCOS:
     RTLIBCASE(COS_F);
+  case TargetOpcode::G_FTAN:
+    RTLIBCASE(TAN_F);
   case TargetOpcode::G_FLOG10:
     RTLIBCASE(LOG10_F);
   case TargetOpcode::G_FLOG:
@@ -1037,6 +1039,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
   case TargetOpcode::G_FREM:
   case TargetOpcode::G_FCOS:
   case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FTAN:
   case TargetOpcode::G_FLOG10:
   case TargetOpcode::G_FLOG:
   case TargetOpcode::G_FLOG2:
@@ -2893,6 +2896,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_FFLOOR:
   case TargetOpcode::G_FCOS:
   case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FTAN:
   case TargetOpcode::G_FLOG10:
   case TargetOpcode::G_FLOG:
   case TargetOpcode::G_FLOG2:
@@ -4659,6 +4663,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_INTRINSIC_TRUNC:
   case G_FCOS:
   case G_FSIN:
+  case G_FTAN:
   case G_FSQRT:
   case G_BSWAP:
   case G_BITREVERSE:
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index afe27035694070bd6702473688830b8a4016ddd0..37aa4e0aed7e822d44ac1deda65a0d8491b4aa50 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -996,7 +996,6 @@ MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
   LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
   LLT AddrTy = Addr.getLLTTy(*getMRI());
   LLT ValTy = Val.getLLTTy(*getMRI());
-  assert(OldValResTy.isScalar() && "invalid operand type");
   assert(AddrTy.isPointer() && "invalid operand type");
   assert(ValTy.isValid() && "invalid operand type");
   assert(OldValResTy == ValTy && "type mismatch");
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index e8438be94b3cd206e3c5bfb3b8bc27bd5c55c7f1..129e6963aef33e01f79ad7af92637a95d2d2aa01 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -833,6 +833,7 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
   case TargetOpcode::G_FREM:
   case TargetOpcode::G_FSIN:
   case TargetOpcode::G_FCOS:
+  case TargetOpcode::G_FTAN:
   case TargetOpcode::G_FMA:
   case TargetOpcode::G_FMAD:
     if (SNaN)
@@ -1713,6 +1714,7 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
   case TargetOpcode::G_FREM:
   case TargetOpcode::G_FRINT:
   case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FTAN:
   case TargetOpcode::G_FSQRT:
   case TargetOpcode::G_FSUB:
   case TargetOpcode::G_INTRINSIC_ROUND:
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index 626e577a30bf320794e2210719b6f4854b30140c..9e2e6316dc6d12c3fd80c479058b4ce83ac98a76 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -828,10 +828,12 @@ bool MachineOutliner::outline(Module &M,
                     << "\n");
   bool OutlinedSomething = false;
 
-  // Sort by benefit. The most beneficial functions should be outlined first.
+  // Sort by priority where priority := getNotOutlinedCost / getOutliningCost.
+  // The function with highest priority should be outlined first.
   stable_sort(FunctionList,
               [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) {
-                return LHS.getBenefit() > RHS.getBenefit();
+                return LHS.getNotOutlinedCost() * RHS.getOutliningCost() >
+                       RHS.getNotOutlinedCost() * LHS.getOutliningCost();
               });
 
   // Walk over each function, outlining them as we go along. Functions are
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 6740e1f0edb4f465e119e4eba9f9d5da2a6347de..09ce8c42a3850dac32d17c4fcacb4bb811c4e888 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -12,6 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/RegAllocFast.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IndexedMap.h"
@@ -174,14 +175,12 @@ private:
   DenseMap<const MachineInstr *, uint64_t> Instr2PosIndex;
 };
 
-class RegAllocFast : public MachineFunctionPass {
+class RegAllocFastImpl {
 public:
-  static char ID;
-
-  RegAllocFast(const RegClassFilterFunc F = allocateAllRegClasses,
-               bool ClearVirtRegs_ = true)
-      : MachineFunctionPass(ID), ShouldAllocateClass(F),
-        StackSlotForVirtReg(-1), ClearVirtRegs(ClearVirtRegs_) {}
+  RegAllocFastImpl(const RegClassFilterFunc F = allocateAllRegClasses,
+                   bool ClearVirtRegs_ = true)
+      : ShouldAllocateClass(F), StackSlotForVirtReg(-1),
+        ClearVirtRegs(ClearVirtRegs_) {}
 
 private:
   MachineFrameInfo *MFI = nullptr;
@@ -197,8 +196,6 @@ private:
   /// Maps virtual regs to the frame index where these values are spilled.
   IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;
 
-  bool ClearVirtRegs;
-
   /// Everything we know about a live virtual register.
   struct LiveReg {
     MachineInstr *LastUse = nullptr; ///< Last instr to use reg.
@@ -318,35 +315,11 @@ private:
   };
 
 public:
-  StringRef getPassName() const override { return "Fast Register Allocator"; }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  MachineFunctionProperties getRequiredProperties() const override {
-    return MachineFunctionProperties().set(
-        MachineFunctionProperties::Property::NoPHIs);
-  }
-
-  MachineFunctionProperties getSetProperties() const override {
-    if (ClearVirtRegs) {
-      return MachineFunctionProperties().set(
-          MachineFunctionProperties::Property::NoVRegs);
-    }
-
-    return MachineFunctionProperties();
-  }
+  bool ClearVirtRegs;
 
-  MachineFunctionProperties getClearedProperties() const override {
-    return MachineFunctionProperties().set(
-        MachineFunctionProperties::Property::IsSSA);
-  }
+  bool runOnMachineFunction(MachineFunction &MF);
 
 private:
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
   void allocateBasicBlock(MachineBasicBlock &MBB);
 
   void addRegClassDefCounts(std::vector<unsigned> &RegClassDefCounts,
@@ -408,6 +381,47 @@ private:
   void dumpState() const;
 };
 
+class RegAllocFast : public MachineFunctionPass {
+  RegAllocFastImpl Impl;
+
+public:
+  static char ID;
+
+  RegAllocFast(const RegClassFilterFunc F = allocateAllRegClasses,
+               bool ClearVirtRegs_ = true)
+      : MachineFunctionPass(ID), Impl(F, ClearVirtRegs_) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    return Impl.runOnMachineFunction(MF);
+  }
+
+  StringRef getPassName() const override { return "Fast Register Allocator"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+
+  MachineFunctionProperties getSetProperties() const override {
+    if (Impl.ClearVirtRegs) {
+      return MachineFunctionProperties().set(
+          MachineFunctionProperties::Property::NoVRegs);
+    }
+
+    return MachineFunctionProperties();
+  }
+
+  MachineFunctionProperties getClearedProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+};
+
 } // end anonymous namespace
 
 char RegAllocFast::ID = 0;
@@ -415,18 +429,18 @@ char RegAllocFast::ID = 0;
 INITIALIZE_PASS(RegAllocFast, "regallocfast", "Fast Register Allocator", false,
                 false)
 
-bool RegAllocFast::shouldAllocateRegister(const Register Reg) const {
+bool RegAllocFastImpl::shouldAllocateRegister(const Register Reg) const {
   assert(Reg.isVirtual());
   const TargetRegisterClass &RC = *MRI->getRegClass(Reg);
   return ShouldAllocateClass(*TRI, RC);
 }
 
-void RegAllocFast::setPhysRegState(MCPhysReg PhysReg, unsigned NewState) {
+void RegAllocFastImpl::setPhysRegState(MCPhysReg PhysReg, unsigned NewState) {
   for (MCRegUnit Unit : TRI->regunits(PhysReg))
     RegUnitStates[Unit] = NewState;
 }
 
-bool RegAllocFast::isPhysRegFree(MCPhysReg PhysReg) const {
+bool RegAllocFastImpl::isPhysRegFree(MCPhysReg PhysReg) const {
   for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
     if (RegUnitStates[Unit] != regFree)
       return false;
@@ -436,7 +450,7 @@ bool RegAllocFast::isPhysRegFree(MCPhysReg PhysReg) const {
 
 /// This allocates space for the specified virtual register to be held on the
 /// stack.
-int RegAllocFast::getStackSpaceFor(Register VirtReg) {
+int RegAllocFastImpl::getStackSpaceFor(Register VirtReg) {
   // Find the location Reg would belong...
   int SS = StackSlotForVirtReg[VirtReg];
   // Already has space allocated?
@@ -464,7 +478,7 @@ static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A,
 }
 
 /// Returns false if \p VirtReg is known to not live out of the current block.
-bool RegAllocFast::mayLiveOut(Register VirtReg) {
+bool RegAllocFastImpl::mayLiveOut(Register VirtReg) {
   if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg))) {
     // Cannot be live-out if there are no successors.
     return !MBB->succ_empty();
@@ -517,7 +531,7 @@ bool RegAllocFast::mayLiveOut(Register VirtReg) {
 }
 
 /// Returns false if \p VirtReg is known to not be live into the current block.
-bool RegAllocFast::mayLiveIn(Register VirtReg) {
+bool RegAllocFastImpl::mayLiveIn(Register VirtReg) {
   if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg)))
     return !MBB->pred_empty();
 
@@ -536,8 +550,9 @@ bool RegAllocFast::mayLiveIn(Register VirtReg) {
 
 /// Insert spill instruction for \p AssignedReg before \p Before. Update
 /// DBG_VALUEs with \p VirtReg operands with the stack slot.
-void RegAllocFast::spill(MachineBasicBlock::iterator Before, Register VirtReg,
-                         MCPhysReg AssignedReg, bool Kill, bool LiveOut) {
+void RegAllocFastImpl::spill(MachineBasicBlock::iterator Before,
+                             Register VirtReg, MCPhysReg AssignedReg, bool Kill,
+                             bool LiveOut) {
   LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " in "
                     << printReg(AssignedReg, TRI));
   int FI = getStackSpaceFor(VirtReg);
@@ -596,8 +611,8 @@ void RegAllocFast::spill(MachineBasicBlock::iterator Before, Register VirtReg,
 }
 
 /// Insert reload instruction for \p PhysReg before \p Before.
-void RegAllocFast::reload(MachineBasicBlock::iterator Before, Register VirtReg,
-                          MCPhysReg PhysReg) {
+void RegAllocFastImpl::reload(MachineBasicBlock::iterator Before,
+                              Register VirtReg, MCPhysReg PhysReg) {
   LLVM_DEBUG(dbgs() << "Reloading " << printReg(VirtReg, TRI) << " into "
                     << printReg(PhysReg, TRI) << '\n');
   int FI = getStackSpaceFor(VirtReg);
@@ -610,7 +625,7 @@ void RegAllocFast::reload(MachineBasicBlock::iterator Before, Register VirtReg,
 /// This is not just MBB.begin() because surprisingly we have EH_LABEL
 /// instructions marking the begin of a basic block. This means we must insert
 /// new instructions after such labels...
-MachineBasicBlock::iterator RegAllocFast::getMBBBeginInsertionPoint(
+MachineBasicBlock::iterator RegAllocFastImpl::getMBBBeginInsertionPoint(
     MachineBasicBlock &MBB, SmallSet<Register, 2> &PrologLiveIns) const {
   MachineBasicBlock::iterator I = MBB.begin();
   while (I != MBB.end()) {
@@ -637,7 +652,7 @@ MachineBasicBlock::iterator RegAllocFast::getMBBBeginInsertionPoint(
 }
 
 /// Reload all currently assigned virtual registers.
-void RegAllocFast::reloadAtBegin(MachineBasicBlock &MBB) {
+void RegAllocFastImpl::reloadAtBegin(MachineBasicBlock &MBB) {
   if (LiveVirtRegs.empty())
     return;
 
@@ -680,7 +695,7 @@ void RegAllocFast::reloadAtBegin(MachineBasicBlock &MBB) {
 /// Handle the direct use of a physical register.  Check that the register is
 /// not used by a virtreg. Kill the physreg, marking it free. This may add
 /// implicit kills to MO->getParent() and invalidate MO.
-bool RegAllocFast::usePhysReg(MachineInstr &MI, MCPhysReg Reg) {
+bool RegAllocFastImpl::usePhysReg(MachineInstr &MI, MCPhysReg Reg) {
   assert(Register::isPhysicalRegister(Reg) && "expected physreg");
   bool displacedAny = displacePhysReg(MI, Reg);
   setPhysRegState(Reg, regPreAssigned);
@@ -688,7 +703,7 @@ bool RegAllocFast::usePhysReg(MachineInstr &MI, MCPhysReg Reg) {
   return displacedAny;
 }
 
-bool RegAllocFast::definePhysReg(MachineInstr &MI, MCPhysReg Reg) {
+bool RegAllocFastImpl::definePhysReg(MachineInstr &MI, MCPhysReg Reg) {
   bool displacedAny = displacePhysReg(MI, Reg);
   setPhysRegState(Reg, regPreAssigned);
   return displacedAny;
@@ -697,7 +712,7 @@ bool RegAllocFast::definePhysReg(MachineInstr &MI, MCPhysReg Reg) {
 /// Mark PhysReg as reserved or free after spilling any virtregs. This is very
 /// similar to defineVirtReg except the physreg is reserved instead of
 /// allocated.
-bool RegAllocFast::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) {
+bool RegAllocFastImpl::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) {
   bool displacedAny = false;
 
   for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
@@ -726,7 +741,7 @@ bool RegAllocFast::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) {
   return displacedAny;
 }
 
-void RegAllocFast::freePhysReg(MCPhysReg PhysReg) {
+void RegAllocFastImpl::freePhysReg(MCPhysReg PhysReg) {
   LLVM_DEBUG(dbgs() << "Freeing " << printReg(PhysReg, TRI) << ':');
 
   MCRegister FirstUnit = *TRI->regunits(PhysReg).begin();
@@ -753,7 +768,7 @@ void RegAllocFast::freePhysReg(MCPhysReg PhysReg) {
 /// for allocation. Returns 0 when PhysReg is free or disabled with all aliases
 /// disabled - it can be allocated directly.
 /// \returns spillImpossible when PhysReg or an alias can't be spilled.
-unsigned RegAllocFast::calcSpillCost(MCPhysReg PhysReg) const {
+unsigned RegAllocFastImpl::calcSpillCost(MCPhysReg PhysReg) const {
   for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
     switch (unsigned VirtReg = RegUnitStates[Unit]) {
     case regFree:
@@ -772,8 +787,9 @@ unsigned RegAllocFast::calcSpillCost(MCPhysReg PhysReg) const {
   return 0;
 }
 
-void RegAllocFast::assignDanglingDebugValues(MachineInstr &Definition,
-                                             Register VirtReg, MCPhysReg Reg) {
+void RegAllocFastImpl::assignDanglingDebugValues(MachineInstr &Definition,
+                                                 Register VirtReg,
+                                                 MCPhysReg Reg) {
   auto UDBGValIter = DanglingDbgValues.find(VirtReg);
   if (UDBGValIter == DanglingDbgValues.end())
     return;
@@ -809,8 +825,8 @@ void RegAllocFast::assignDanglingDebugValues(MachineInstr &Definition,
 /// This method updates local state so that we know that PhysReg is the
 /// proper container for VirtReg now.  The physical register must not be used
 /// for anything else when this is called.
-void RegAllocFast::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR,
-                                       MCPhysReg PhysReg) {
+void RegAllocFastImpl::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR,
+                                           MCPhysReg PhysReg) {
   Register VirtReg = LR.VirtReg;
   LLVM_DEBUG(dbgs() << "Assigning " << printReg(VirtReg, TRI) << " to "
                     << printReg(PhysReg, TRI) << '\n');
@@ -824,7 +840,7 @@ void RegAllocFast::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR,
 
 static bool isCoalescable(const MachineInstr &MI) { return MI.isFullCopy(); }
 
-Register RegAllocFast::traceCopyChain(Register Reg) const {
+Register RegAllocFastImpl::traceCopyChain(Register Reg) const {
   static const unsigned ChainLengthLimit = 3;
   unsigned C = 0;
   do {
@@ -843,7 +859,7 @@ Register RegAllocFast::traceCopyChain(Register Reg) const {
 /// Check if any of \p VirtReg's definitions is a copy. If it is follow the
 /// chain of copies to check whether we reach a physical register we can
 /// coalesce with.
-Register RegAllocFast::traceCopies(Register VirtReg) const {
+Register RegAllocFastImpl::traceCopies(Register VirtReg) const {
   static const unsigned DefLimit = 3;
   unsigned C = 0;
   for (const MachineInstr &MI : MRI->def_instructions(VirtReg)) {
@@ -861,8 +877,8 @@ Register RegAllocFast::traceCopies(Register VirtReg) const {
 }
 
 /// Allocates a physical register for VirtReg.
-void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, Register Hint0,
-                                bool LookAtPhysRegUses) {
+void RegAllocFastImpl::allocVirtReg(MachineInstr &MI, LiveReg &LR,
+                                    Register Hint0, bool LookAtPhysRegUses) {
   const Register VirtReg = LR.VirtReg;
   assert(LR.PhysReg == 0);
 
@@ -950,7 +966,7 @@ void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, Register Hint0,
   assignVirtToPhysReg(MI, LR, BestReg);
 }
 
-void RegAllocFast::allocVirtRegUndef(MachineOperand &MO) {
+void RegAllocFastImpl::allocVirtRegUndef(MachineOperand &MO) {
   assert(MO.isUndef() && "expected undef use");
   Register VirtReg = MO.getReg();
   assert(VirtReg.isVirtual() && "Expected virtreg");
@@ -980,8 +996,9 @@ void RegAllocFast::allocVirtRegUndef(MachineOperand &MO) {
 /// Variation of defineVirtReg() with special handling for livethrough regs
 /// (tied or earlyclobber) that may interfere with preassigned uses.
 /// \return true if MI's MachineOperands were re-arranged/invalidated.
-bool RegAllocFast::defineLiveThroughVirtReg(MachineInstr &MI, unsigned OpNum,
-                                            Register VirtReg) {
+bool RegAllocFastImpl::defineLiveThroughVirtReg(MachineInstr &MI,
+                                                unsigned OpNum,
+                                                Register VirtReg) {
   if (!shouldAllocateRegister(VirtReg))
     return false;
   LiveRegMap::iterator LRI = findLiveVirtReg(VirtReg);
@@ -1016,8 +1033,8 @@ bool RegAllocFast::defineLiveThroughVirtReg(MachineInstr &MI, unsigned OpNum,
 /// - The value is live out and all uses are in different basic blocks.
 ///
 /// \return true if MI's MachineOperands were re-arranged/invalidated.
-bool RegAllocFast::defineVirtReg(MachineInstr &MI, unsigned OpNum,
-                                 Register VirtReg, bool LookAtPhysRegUses) {
+bool RegAllocFastImpl::defineVirtReg(MachineInstr &MI, unsigned OpNum,
+                                     Register VirtReg, bool LookAtPhysRegUses) {
   assert(VirtReg.isVirtual() && "Not a virtual register");
   if (!shouldAllocateRegister(VirtReg))
     return false;
@@ -1094,8 +1111,8 @@ bool RegAllocFast::defineVirtReg(MachineInstr &MI, unsigned OpNum,
 
 /// Allocates a register for a VirtReg use.
 /// \return true if MI's MachineOperands were re-arranged/invalidated.
-bool RegAllocFast::useVirtReg(MachineInstr &MI, MachineOperand &MO,
-                              Register VirtReg) {
+bool RegAllocFastImpl::useVirtReg(MachineInstr &MI, MachineOperand &MO,
+                                  Register VirtReg) {
   assert(VirtReg.isVirtual() && "Not a virtual register");
   if (!shouldAllocateRegister(VirtReg))
     return false;
@@ -1150,8 +1167,8 @@ bool RegAllocFast::useVirtReg(MachineInstr &MI, MachineOperand &MO,
 
 /// Changes operand OpNum in MI the refer the PhysReg, considering subregs.
 /// \return true if MI's MachineOperands were re-arranged/invalidated.
-bool RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO,
-                              MCPhysReg PhysReg) {
+bool RegAllocFastImpl::setPhysReg(MachineInstr &MI, MachineOperand &MO,
+                                  MCPhysReg PhysReg) {
   if (!MO.getSubReg()) {
     MO.setReg(PhysReg);
     MO.setIsRenamable(true);
@@ -1190,7 +1207,7 @@ bool RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO,
 
 #ifndef NDEBUG
 
-void RegAllocFast::dumpState() const {
+void RegAllocFastImpl::dumpState() const {
   for (unsigned Unit = 1, UnitE = TRI->getNumRegUnits(); Unit != UnitE;
        ++Unit) {
     switch (unsigned VirtReg = RegUnitStates[Unit]) {
@@ -1235,7 +1252,7 @@ void RegAllocFast::dumpState() const {
 #endif
 
 /// Count number of defs consumed from each register class by \p Reg
-void RegAllocFast::addRegClassDefCounts(
+void RegAllocFastImpl::addRegClassDefCounts(
     std::vector<unsigned> &RegClassDefCounts, Register Reg) const {
   assert(RegClassDefCounts.size() == TRI->getNumRegClasses());
 
@@ -1269,7 +1286,7 @@ void RegAllocFast::addRegClassDefCounts(
 /// Compute \ref DefOperandIndexes so it contains the indices of "def" operands
 /// that are to be allocated. Those are ordered in a way that small classes,
 /// early clobbers and livethroughs are allocated first.
-void RegAllocFast::findAndSortDefOperandIndexes(const MachineInstr &MI) {
+void RegAllocFastImpl::findAndSortDefOperandIndexes(const MachineInstr &MI) {
   DefOperandIndexes.clear();
 
   // Track number of defs which may consume a register from the class.
@@ -1343,7 +1360,7 @@ static bool isTiedToNotUndef(const MachineOperand &MO) {
   return !TiedMO.isUndef();
 }
 
-void RegAllocFast::allocateInstruction(MachineInstr &MI) {
+void RegAllocFastImpl::allocateInstruction(MachineInstr &MI) {
   // The basic algorithm here is:
   // 1. Mark registers of def operands as free
   // 2. Allocate registers to use operands and place reload instructions for
@@ -1605,7 +1622,7 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) {
   }
 }
 
-void RegAllocFast::handleDebugValue(MachineInstr &MI) {
+void RegAllocFastImpl::handleDebugValue(MachineInstr &MI) {
   // Ignore DBG_VALUEs that aren't based on virtual registers. These are
   // mostly constants and frame indices.
   assert(MI.isDebugValue() && "not a DBG_VALUE*");
@@ -1648,7 +1665,7 @@ void RegAllocFast::handleDebugValue(MachineInstr &MI) {
   }
 }
 
-void RegAllocFast::handleBundle(MachineInstr &MI) {
+void RegAllocFastImpl::handleBundle(MachineInstr &MI) {
   MachineBasicBlock::instr_iterator BundledMI = MI.getIterator();
   ++BundledMI;
   while (BundledMI->isBundledWithPred()) {
@@ -1671,7 +1688,7 @@ void RegAllocFast::handleBundle(MachineInstr &MI) {
   }
 }
 
-void RegAllocFast::allocateBasicBlock(MachineBasicBlock &MBB) {
+void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) {
   this->MBB = &MBB;
   LLVM_DEBUG(dbgs() << "\nAllocating " << MBB);
 
@@ -1732,7 +1749,7 @@ void RegAllocFast::allocateBasicBlock(MachineBasicBlock &MBB) {
   LLVM_DEBUG(MBB.dump());
 }
 
-bool RegAllocFast::runOnMachineFunction(MachineFunction &MF) {
+bool RegAllocFastImpl::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** FAST REGISTER ALLOCATION **********\n"
                     << "********** Function: " << MF.getName() << '\n');
   MRI = &MF.getRegInfo();
@@ -1771,6 +1788,36 @@ bool RegAllocFast::runOnMachineFunction(MachineFunction &MF) {
   return true;
 }
 
+PreservedAnalyses RegAllocFastPass::run(MachineFunction &MF,
+                                        MachineFunctionAnalysisManager &) {
+  RegAllocFastImpl Impl(Opts.Filter, Opts.ClearVRegs);
+  bool Changed = Impl.runOnMachineFunction(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+void RegAllocFastPass::printPipeline(
+    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
+  bool PrintFilterName = Opts.FilterName != "all";
+  bool PrintNoClearVRegs = !Opts.ClearVRegs;
+  bool PrintSemicolon = PrintFilterName && PrintNoClearVRegs;
+
+  OS << "regallocfast";
+  if (PrintFilterName || PrintNoClearVRegs) {
+    OS << '<';
+    if (PrintFilterName)
+      OS << "filter=" << Opts.FilterName;
+    if (PrintSemicolon)
+      OS << ';';
+    if (PrintNoClearVRegs)
+      OS << "no-clear-vregs";
+    OS << '>';
+  }
+}
+
 FunctionPass *llvm::createFastRegisterAllocator() { return new RegAllocFast(); }
 
 FunctionPass *llvm::createFastRegisterAllocator(RegClassFilterFunc Ftor,
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 3397bd0a6060311daddbfdcca3632fbdad2b0ff5..a808a541103f5feaa0c1d5cd101ba4966e9b3736 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -1339,14 +1339,13 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
   if (SrcIdx && DstIdx)
     return false;
 
-  [[maybe_unused]] const unsigned DefSubIdx = DefMI->getOperand(0).getSubReg();
+  const unsigned DefSubIdx = DefMI->getOperand(0).getSubReg();
   const TargetRegisterClass *DefRC = TII->getRegClass(MCID, 0, TRI, *MF);
   if (!DefMI->isImplicitDef()) {
     if (DstReg.isPhysical()) {
       Register NewDstReg = DstReg;
 
-      unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(),
-                                              DefMI->getOperand(0).getSubReg());
+      unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), DefSubIdx);
       if (NewDstIdx)
         NewDstReg = TRI->getSubReg(DstReg, NewDstIdx);
 
diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
index c67cc57ba675dea542d0b495a408943006e70fb4..9fbb7b461364b10fd37b8772ff03cb69a7fb2e5a 100644
--- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
+++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Replaces LLVM IR instructions with vector operands (i.e., the frem
-// instruction or calls to LLVM intrinsics) with matching calls to functions
-// from a vector library (e.g libmvec, SVML) using TargetLibraryInfo interface.
+// Replaces calls to LLVM Intrinsics with matching calls to functions from a
+// vector library (e.g libmvec, SVML) using TargetLibraryInfo interface.
 //
 //===----------------------------------------------------------------------===//
 
@@ -25,6 +24,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/VFABIDemangler.h"
 #include "llvm/Support/TypeSize.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -70,84 +70,68 @@ Function *getTLIFunction(Module *M, FunctionType *VectorFTy,
   return TLIFunc;
 }
 
-/// Replace the instruction \p I with a call to the corresponding function from
-/// the vector library (\p TLIVecFunc).
-static void replaceWithTLIFunction(Instruction &I, VFInfo &Info,
+/// Replace the intrinsic call \p II to \p TLIVecFunc, which is the
+/// corresponding function from the vector library.
+static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info,
                                    Function *TLIVecFunc) {
-  IRBuilder<> IRBuilder(&I);
-  auto *CI = dyn_cast<CallInst>(&I);
-  SmallVector<Value *> Args(CI ? CI->args() : I.operands());
+  IRBuilder<> IRBuilder(II);
+  SmallVector<Value *> Args(II->args());
   if (auto OptMaskpos = Info.getParamIndexForOptionalMask()) {
     auto *MaskTy =
-        VectorType::get(Type::getInt1Ty(I.getContext()), Info.Shape.VF);
+        VectorType::get(Type::getInt1Ty(II->getContext()), Info.Shape.VF);
     Args.insert(Args.begin() + OptMaskpos.value(),
                 Constant::getAllOnesValue(MaskTy));
   }
 
-  // If it is a call instruction, preserve the operand bundles.
+  // Preserve the operand bundles.
   SmallVector<OperandBundleDef, 1> OpBundles;
-  if (CI)
-    CI->getOperandBundlesAsDefs(OpBundles);
+  II->getOperandBundlesAsDefs(OpBundles);
 
   auto *Replacement = IRBuilder.CreateCall(TLIVecFunc, Args, OpBundles);
-  I.replaceAllUsesWith(Replacement);
+  II->replaceAllUsesWith(Replacement);
   // Preserve fast math flags for FP math.
   if (isa<FPMathOperator>(Replacement))
-    Replacement->copyFastMathFlags(&I);
+    Replacement->copyFastMathFlags(II);
 }
 
-/// Returns true when successfully replaced \p I with a suitable function taking
-/// vector arguments, based on available mappings in the \p TLI. Currently only
-/// works when \p I is a call to vectorized intrinsic or the frem instruction.
+/// Returns true when successfully replaced \p II, which is a call to a
+/// vectorized intrinsic, with a suitable function taking vector arguments,
+/// based on available mappings in the \p TLI.
 static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
-                                    Instruction &I) {
+                                    IntrinsicInst *II) {
+  assert(II != nullptr && "Intrinsic cannot be null");
   // At the moment VFABI assumes the return type is always widened unless it is
   // a void type.
-  auto *VTy = dyn_cast<VectorType>(I.getType());
+  auto *VTy = dyn_cast<VectorType>(II->getType());
   ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0));
-
-  // Compute the argument types of the corresponding scalar call and the scalar
-  // function name. For calls, it additionally finds the function to replace
-  // and checks that all vector operands match the previously found EC.
+  // Compute the argument types of the corresponding scalar call and check that
+  // all vector operands match the previously found EC.
   SmallVector<Type *, 8> ScalarArgTypes;
-  std::string ScalarName;
-  Function *FuncToReplace = nullptr;
-  auto *CI = dyn_cast<CallInst>(&I);
-  if (CI) {
-    FuncToReplace = CI->getCalledFunction();
-    Intrinsic::ID IID = FuncToReplace->getIntrinsicID();
-    assert(IID != Intrinsic::not_intrinsic && "Not an intrinsic");
-    for (auto Arg : enumerate(CI->args())) {
-      auto *ArgTy = Arg.value()->getType();
-      if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) {
-        ScalarArgTypes.push_back(ArgTy);
-      } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) {
-        ScalarArgTypes.push_back(VectorArgTy->getElementType());
-        // When return type is void, set EC to the first vector argument, and
-        // disallow vector arguments with different ECs.
-        if (EC.isZero())
-          EC = VectorArgTy->getElementCount();
-        else if (EC != VectorArgTy->getElementCount())
-          return false;
-      } else
-        // Exit when it is supposed to be a vector argument but it isn't.
+  Intrinsic::ID IID = II->getIntrinsicID();
+  for (auto Arg : enumerate(II->args())) {
+    auto *ArgTy = Arg.value()->getType();
+    if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) {
+      ScalarArgTypes.push_back(ArgTy);
+    } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) {
+      ScalarArgTypes.push_back(VectorArgTy->getElementType());
+      // When return type is void, set EC to the first vector argument, and
+      // disallow vector arguments with different ECs.
+      if (EC.isZero())
+        EC = VectorArgTy->getElementCount();
+      else if (EC != VectorArgTy->getElementCount())
         return false;
-    }
-    // Try to reconstruct the name for the scalar version of the instruction,
-    // using scalar argument types.
-    ScalarName = Intrinsic::isOverloaded(IID)
-                     ? Intrinsic::getName(IID, ScalarArgTypes, I.getModule())
-                     : Intrinsic::getName(IID).str();
-  } else {
-    assert(VTy && "Return type must be a vector");
-    auto *ScalarTy = VTy->getScalarType();
-    LibFunc Func;
-    if (!TLI.getLibFunc(I.getOpcode(), ScalarTy, Func))
+    } else
+      // Exit when it is supposed to be a vector argument but it isn't.
       return false;
-    ScalarName = TLI.getName(Func);
-    ScalarArgTypes = {ScalarTy, ScalarTy};
   }
 
+  // Try to reconstruct the name for the scalar version of the instruction,
+  // using scalar argument types.
+  std::string ScalarName =
+      Intrinsic::isOverloaded(IID)
+          ? Intrinsic::getName(IID, ScalarArgTypes, II->getModule())
+          : Intrinsic::getName(IID).str();
+
   // Try to find the mapping for the scalar version of this intrinsic and the
   // exact vector width of the call operands in the TargetLibraryInfo. First,
   // check with a non-masked variant, and if that fails try with a masked one.
@@ -162,7 +146,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
 
   // Replace the call to the intrinsic with a call to the vector library
   // function.
-  Type *ScalarRetTy = I.getType()->getScalarType();
+  Type *ScalarRetTy = II->getType()->getScalarType();
   FunctionType *ScalarFTy =
       FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false);
   const std::string MangledName = VD->getVectorFunctionABIVariantString();
@@ -174,22 +158,19 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
   // specification when being created, this is why we need to add extra check to
   // make sure that the operands of the vector function obtained via VFABI match
   // the operands of the original vector instruction.
-  if (CI) {
-    for (auto &VFParam : OptInfo->Shape.Parameters) {
-      if (VFParam.ParamKind == VFParamKind::GlobalPredicate)
-        continue;
+  for (auto &VFParam : OptInfo->Shape.Parameters) {
+    if (VFParam.ParamKind == VFParamKind::GlobalPredicate)
+      continue;
 
-      // tryDemangleForVFABI must return valid ParamPos, otherwise it could be
-      // a bug in the VFABI parser.
-      assert(VFParam.ParamPos < CI->arg_size() &&
-             "ParamPos has invalid range.");
-      Type *OrigTy = CI->getArgOperand(VFParam.ParamPos)->getType();
-      if (OrigTy->isVectorTy() != (VFParam.ParamKind == VFParamKind::Vector)) {
-        LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Will not replace: " << ScalarName
-                          << ". Wrong type at index " << VFParam.ParamPos
-                          << ": " << *OrigTy << "\n");
-        return false;
-      }
+    // tryDemangleForVFABI must return valid ParamPos, otherwise it could be
+    // a bug in the VFABI parser.
+    assert(VFParam.ParamPos < II->arg_size() && "ParamPos has invalid range");
+    Type *OrigTy = II->getArgOperand(VFParam.ParamPos)->getType();
+    if (OrigTy->isVectorTy() != (VFParam.ParamKind == VFParamKind::Vector)) {
+      LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Will not replace: " << ScalarName
+                        << ". Wrong type at index " << VFParam.ParamPos << ": "
+                        << *OrigTy << "\n");
+      return false;
     }
   }
 
@@ -197,45 +178,32 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
   if (!VectorFTy)
     return false;
 
-  Function *TLIFunc = getTLIFunction(I.getModule(), VectorFTy,
-                                     VD->getVectorFnName(), FuncToReplace);
-
-  replaceWithTLIFunction(I, *OptInfo, TLIFunc);
+  Function *TLIFunc =
+      getTLIFunction(II->getModule(), VectorFTy, VD->getVectorFnName(),
+                     II->getCalledFunction());
+  replaceWithTLIFunction(II, *OptInfo, TLIFunc);
   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" << ScalarName
                     << "` with call to `" << TLIFunc->getName() << "`.\n");
   ++NumCallsReplaced;
   return true;
 }
 
-/// Supported instruction \p I must be a vectorized frem or a call to an
-/// intrinsic that returns either void or a vector.
-static bool isSupportedInstruction(Instruction *I) {
-  Type *Ty = I->getType();
-  if (auto *CI = dyn_cast<CallInst>(I))
-    return (Ty->isVectorTy() || Ty->isVoidTy()) && CI->getCalledFunction() &&
-           CI->getCalledFunction()->getIntrinsicID() !=
-               Intrinsic::not_intrinsic;
-  if (I->getOpcode() == Instruction::FRem && Ty->isVectorTy())
-    return true;
-  return false;
-}
-
 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
-  bool Changed = false;
   SmallVector<Instruction *> ReplacedCalls;
   for (auto &I : instructions(F)) {
-    if (!isSupportedInstruction(&I))
-      continue;
-    if (replaceWithCallToVeclib(TLI, I)) {
-      ReplacedCalls.push_back(&I);
-      Changed = true;
+    // Process only intrinsic calls that return void or a vector.
+    if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+      if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
+        continue;
+
+      if (replaceWithCallToVeclib(TLI, II))
+        ReplacedCalls.push_back(&I);
     }
   }
-  // Erase the calls to the intrinsics that have been replaced
-  // with calls to the vector library.
-  for (auto *CI : ReplacedCalls)
-    CI->eraseFromParent();
-  return Changed;
+  // Erase any intrinsic calls that were replaced with vector library calls.
+  for (auto *I : ReplacedCalls)
+    I->eraseFromParent();
+  return !ReplacedCalls.empty();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -246,7 +214,7 @@ PreservedAnalyses ReplaceWithVeclib::run(Function &F,
   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto Changed = runImpl(TLI, F);
   if (Changed) {
-    LLVM_DEBUG(dbgs() << "Instructions replaced with vector libraries: "
+    LLVM_DEBUG(dbgs() << "Intrinsic calls replaced with vector libraries: "
                       << NumCallsReplaced << "\n");
 
     PreservedAnalyses PA;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9a5359015439e054aa2878c8301ab30bdf4662e5..4fcbe08e4b2b98d52c9e572ea23ec0280c1db7ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -180,12 +180,6 @@ namespace {
     /// in the worklist, this is different from the tail of the worklist.
     SmallSetVector<SDNode *, 32> PruningList;
 
-    /// Set of nodes which have been combined (at least once).
-    ///
-    /// This is used to allow us to reliably add any operands of a DAG node
-    /// which have not yet been combined to the worklist.
-    SmallPtrSet<SDNode *, 32> CombinedNodes;
-
     /// Map from candidate StoreNode to the pair of RootNode and count.
     /// The count is used to track how many times we have seen the StoreNode
     /// with the same RootNode bail out in dependence check. If we have seen
@@ -235,7 +229,8 @@ namespace {
       if (N) {
         assert(N->getCombinerWorklistIndex() >= 0 &&
                "Found a worklist entry without a corresponding map entry!");
-        N->setCombinerWorklistIndex(-1);
+        // Set to -2 to indicate that we combined the node.
+        N->setCombinerWorklistIndex(-2);
       }
       return N;
     }
@@ -267,7 +262,8 @@ namespace {
 
     /// Add to the worklist making sure its instance is at the back (next to be
     /// processed.)
-    void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true) {
+    void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true,
+                       bool SkipIfCombinedBefore = false) {
       assert(N->getOpcode() != ISD::DELETED_NODE &&
              "Deleted Node added to Worklist");
 
@@ -276,10 +272,13 @@ namespace {
       if (N->getOpcode() == ISD::HANDLENODE)
         return;
 
+      if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2)
+        return;
+
       if (IsCandidateForPruning)
         ConsiderForPruning(N);
 
-      if (N->getCombinerWorklistIndex() == -1) {
+      if (N->getCombinerWorklistIndex() < 0) {
         N->setCombinerWorklistIndex(Worklist.size());
         Worklist.push_back(N);
       }
@@ -287,12 +286,14 @@ namespace {
 
     /// Remove all instances of N from the worklist.
     void removeFromWorklist(SDNode *N) {
-      CombinedNodes.erase(N);
       PruningList.remove(N);
       StoreRootCountMap.erase(N);
 
       int WorklistIndex = N->getCombinerWorklistIndex();
-      if (WorklistIndex == -1)
+      // If not in the worklist, the index might be -1 or -2 (was combined
+      // before). As the node gets deleted anyway, there's no need to update
+      // the index.
+      if (WorklistIndex < 0)
         return; // Not in the worklist.
 
       // Null out the entry rather than erasing it to avoid a linear operation.
@@ -1768,13 +1769,13 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
     LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
 
     // Add any operands of the new node which have not yet been combined to the
-    // worklist as well. Because the worklist uniques things already, this
-    // won't repeatedly process the same operand.
+    // worklist as well. getNextWorklistEntry flags nodes that have been
+    // combined before. Because the worklist uniques things already, this won't
+    // repeatedly process the same operand.
     for (const SDValue &ChildN : N->op_values())
-      if (!CombinedNodes.count(ChildN.getNode()))
-        AddToWorklist(ChildN.getNode());
+      AddToWorklist(ChildN.getNode(), /*IsCandidateForPruning=*/true,
+                    /*SkipIfCombinedBefore=*/true);
 
-    CombinedNodes.insert(N);
     SDValue RV = combine(N);
 
     if (!RV.getNode())
@@ -4041,17 +4042,11 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
   }
 
-  // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
-  if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
-    if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
-      SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
-      SDValue S0 = N1.getOperand(0);
-      if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
-        if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
-          if (C->getAPIntValue() == (BitWidth - 1))
-            return DAG.getNode(ISD::ABS, DL, VT, S0);
-    }
-  }
+  // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A)
+  if ((!LegalOperations || hasOperation(ISD::ABS, VT)) &&
+      sd_match(N1, m_Sra(m_Value(A), m_SpecificInt(BitWidth - 1))) &&
+      sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1))))
+    return DAG.getNode(ISD::ABS, DL, VT, A);
 
   // If the relocation model supports it, consider symbol offsets.
   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
@@ -9532,7 +9527,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   }
 
   // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
-  if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
+  if (!LegalOperations || hasOperation(ISD::ABS, VT)) {
     SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
     SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
     if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
@@ -10117,7 +10112,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   // fold (shl X, cttz(Y)) -> (mul (Y & -Y), X) if cttz is unsupported on the
   // target.
   if (((N1.getOpcode() == ISD::CTTZ &&
-        VT.getScalarSizeInBits() >= ShiftVT.getScalarSizeInBits()) ||
+        VT.getScalarSizeInBits() <= ShiftVT.getScalarSizeInBits()) ||
        N1.getOpcode() == ISD::CTTZ_ZERO_UNDEF) &&
       N1.hasOneUse() && !TLI.isOperationLegalOrCustom(ISD::CTTZ, ShiftVT) &&
       TLI.isOperationLegalOrCustom(ISD::MUL, VT)) {
@@ -17267,26 +17262,29 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
   if (SDValue V = combineRepeatedFPDivisors(N))
     return V;
 
-  if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
-    // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
-    if (auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
-      // Compute the reciprocal 1.0 / c2.
-      const APFloat &N1APF = N1CFP->getValueAPF();
-      APFloat Recip(N1APF.getSemantics(), 1); // 1.0
-      APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven);
-      // Only do the transform if the reciprocal is a legal fp immediate that
-      // isn't too nasty (eg NaN, denormal, ...).
-      if ((st == APFloat::opOK || st == APFloat::opInexact) && // Not too nasty
-          (!LegalOperations ||
-           // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
-           // backend)... we should handle this gracefully after Legalize.
-           // TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT) ||
-           TLI.isOperationLegal(ISD::ConstantFP, VT) ||
-           TLI.isFPImmLegal(Recip, VT, ForCodeSize)))
-        return DAG.getNode(ISD::FMUL, DL, VT, N0,
-                           DAG.getConstantFP(Recip, DL, VT));
-    }
+  // fold (fdiv X, c2) -> (fmul X, 1/c2) if there is no loss in precision, or
+  // the loss is acceptable with AllowReciprocal.
+  if (auto *N1CFP = isConstOrConstSplatFP(N1, true)) {
+    // Compute the reciprocal 1.0 / c2.
+    const APFloat &N1APF = N1CFP->getValueAPF();
+    APFloat Recip = APFloat::getOne(N1APF.getSemantics());
+    APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven);
+    // Only do the transform if the reciprocal is a legal fp immediate that
+    // isn't too nasty (eg NaN, denormal, ...).
+    if (((st == APFloat::opOK && !Recip.isDenormal()) ||
+         (st == APFloat::opInexact &&
+          (Options.UnsafeFPMath || Flags.hasAllowReciprocal()))) &&
+        (!LegalOperations ||
+         // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
+         // backend)... we should handle this gracefully after Legalize.
+         // TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT) ||
+         TLI.isOperationLegal(ISD::ConstantFP, VT) ||
+         TLI.isFPImmLegal(Recip, VT, ForCodeSize)))
+      return DAG.getNode(ISD::FMUL, DL, VT, N0,
+                         DAG.getConstantFP(Recip, DL, VT));
+  }
 
+  if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
     // If this FDIV is part of a reciprocal square root, it may be folded
     // into a target-specific square root estimate instruction.
     if (N1.getOpcode() == ISD::FSQRT) {
@@ -26591,7 +26589,12 @@ SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) {
     }
   }
 
-  return SDValue();
+  // Sometimes constants manage to survive very late in the pipeline, e.g.,
+  // because they are wrapped inside the <1 x f16> type. Try one last time to
+  // get rid of them.
+  SDValue Folded = DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N),
+                                              N->getValueType(0), {N0});
+  return Folded;
 }
 
 SDValue DAGCombiner::visitFP_TO_BF16(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 8cd2bb60d81f22d94a40098b419676fc406e4674..27c45cab2e0d03684876fe7c128db0a7c7aed39b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4514,6 +4514,11 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
                     RTLIB::COS_F80, RTLIB::COS_F128,
                     RTLIB::COS_PPCF128, Results);
     break;
+  case ISD::FTAN:
+  case ISD::STRICT_FTAN:
+    ExpandFPLibCall(Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80,
+                    RTLIB::TAN_F128, RTLIB::TAN_PPCF128, Results);
+    break;
   case ISD::FSINCOS:
     // Expand into sincos libcall.
     ExpandSinCosLibCall(Node, Results);
@@ -5468,6 +5473,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::FSQRT:
   case ISD::FSIN:
   case ISD::FCOS:
+  case ISD::FTAN:
   case ISD::FLOG:
   case ISD::FLOG2:
   case ISD::FLOG10:
@@ -5492,6 +5498,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::STRICT_FSQRT:
   case ISD::STRICT_FSIN:
   case ISD::STRICT_FCOS:
+  case ISD::STRICT_FTAN:
   case ISD::STRICT_FLOG:
   case ISD::STRICT_FLOG2:
   case ISD::STRICT_FLOG10:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index fb1424f75e097d3a2351ead995549b54595b0fac..aa116c9de5d8c4935825df93ac3a62634afcc191 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -131,6 +131,8 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FSQRT:       R = SoftenFloatRes_FSQRT(N); break;
     case ISD::STRICT_FSUB:
     case ISD::FSUB:        R = SoftenFloatRes_FSUB(N); break;
+    case ISD::STRICT_FTAN:
+    case ISD::FTAN:        R = SoftenFloatRes_FTAN(N); break;
     case ISD::STRICT_FTRUNC:
     case ISD::FTRUNC:      R = SoftenFloatRes_FTRUNC(N); break;
     case ISD::LOAD:        R = SoftenFloatRes_LOAD(N); break;
@@ -774,6 +776,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
                                                RTLIB::SUB_PPCF128));
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
+  return SoftenFloatRes_Unary(
+      N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
+                      RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
+}
+
 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
   return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
                                               RTLIB::TRUNC_F32,
@@ -1330,7 +1338,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
 #endif
     report_fatal_error("Do not know how to expand the result of this "
                        "operator!");
-
+    // clang-format off
   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
   case ISD::SELECT:       SplitRes_Select(N, Lo, Hi); break;
   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
@@ -1399,6 +1407,8 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
   case ISD::FSQRT:      ExpandFloatRes_FSQRT(N, Lo, Hi); break;
   case ISD::STRICT_FSUB:
   case ISD::FSUB:       ExpandFloatRes_FSUB(N, Lo, Hi); break;
+  case ISD::STRICT_FTAN:
+  case ISD::FTAN:       ExpandFloatRes_FTAN(N, Lo, Hi); break;
   case ISD::STRICT_FTRUNC:
   case ISD::FTRUNC:     ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
   case ISD::LOAD:       ExpandFloatRes_LOAD(N, Lo, Hi); break;
@@ -1408,6 +1418,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
   case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
   case ISD::STRICT_FREM:
   case ISD::FREM:       ExpandFloatRes_FREM(N, Lo, Hi); break;
+    // clang-format on
   }
 
   // If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -1768,6 +1779,15 @@ void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
                                         RTLIB::SUB_PPCF128), Lo, Hi);
 }
 
+void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
+                                           SDValue &Hi) {
+  ExpandFloatRes_Unary(N,
+                       GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
+                                    RTLIB::TAN_F64, RTLIB::TAN_F80,
+                                    RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
+                       Lo, Hi);
+}
+
 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
                                              SDValue &Lo, SDValue &Hi) {
   ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
@@ -2479,6 +2499,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FSIN:
     case ISD::FSQRT:
     case ISD::FTRUNC:
+    case ISD::FTAN:
     case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
 
     // Binary FP Operations
@@ -2914,6 +2935,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
   case ISD::FSIN:
   case ISD::FSQRT:
   case ISD::FTRUNC:
+  case ISD::FTAN:
   case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
 
   // Binary FP Operations
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index bec9cb49b58649e4ca1b4f66b2ffc8200056df39..2350b562a0343d1bb3c9675aae742d6c2b0a13bf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -586,6 +586,7 @@ private:
   SDValue SoftenFloatRes_FSIN(SDNode *N);
   SDValue SoftenFloatRes_FSQRT(SDNode *N);
   SDValue SoftenFloatRes_FSUB(SDNode *N);
+  SDValue SoftenFloatRes_FTAN(SDNode *N);
   SDValue SoftenFloatRes_FTRUNC(SDNode *N);
   SDValue SoftenFloatRes_LOAD(SDNode *N);
   SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);
@@ -635,6 +636,7 @@ private:
                             SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
                              SDValue &Lo, SDValue &Hi);
+  // clang-format off
   void ExpandFloatRes_FABS      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FMINNUM   (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FMAXNUM   (SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -667,9 +669,11 @@ private:
   void ExpandFloatRes_FSIN      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FSQRT     (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FSUB      (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandFloatRes_FTAN      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FTRUNC    (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_LOAD      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
+  // clang-format on
 
   // Float Operand Expansion.
   bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 6acbc044d6731fef376b154c40219f5e8cf98e56..8cdb4ba0ade6fde58467a477672e375aba50339a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -397,6 +397,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::FSQRT:
   case ISD::FSIN:
   case ISD::FCOS:
+  case ISD::FTAN:
   case ISD::FLDEXP:
   case ISD::FPOWI:
   case ISD::FPOW:
@@ -506,7 +507,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
         break;                                                                 \
     }                                                                          \
     /* Defer non-vector results to LegalizeDAG. */                             \
-    if (!Node->getValueType(0).isVector()) {                                   \
+    if (!Node->getValueType(0).isVector() &&                                   \
+        Node->getValueType(0) != MVT::Other) {                                 \
       Action = TargetLowering::Legal;                                          \
       break;                                                                   \
     }                                                                          \
@@ -990,11 +992,8 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     break;
   case ISD::FMINIMUM:
   case ISD::FMAXIMUM:
-    if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)) {
-      Results.push_back(Expanded);
-      return;
-    }
-    break;
+    Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG));
+    return;
   case ISD::SMIN:
   case ISD::SMAX:
   case ISD::UMIN:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 361416edb554caf22e81e304b02aecda64d894a3..92ce3b17ed6c9afd7bcef092575b1ebcffb417a5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -108,6 +108,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FROUNDEVEN:
   case ISD::FSIN:
   case ISD::FSQRT:
+  case ISD::FTAN:
   case ISD::FTRUNC:
   case ISD::SIGN_EXTEND:
   case ISD::SINT_TO_FP:
@@ -1140,6 +1141,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_FROUNDEVEN:
   case ISD::FSIN:
   case ISD::FSQRT: case ISD::VP_SQRT:
+  case ISD::FTAN:
   case ISD::FTRUNC:
   case ISD::VP_FROUNDTOZERO:
   case ISD::SINT_TO_FP:
@@ -4400,6 +4402,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FROUNDEVEN:
   case ISD::FSIN:
   case ISD::FSQRT:
+  case ISD::FTAN:
   case ISD::FTRUNC:
     if (unrollExpandedOp())
       break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 4a6a431696b5254f963820f8a63c5983e52a342e..523d3aea66225206a82f913db55978d8003dca55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4212,7 +4212,6 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     break;
   }
 
-  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
   return Known;
 }
 
@@ -5375,6 +5374,7 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
   case ISD::FREM:
   case ISD::FSIN:
   case ISD::FCOS:
+  case ISD::FTAN:
   case ISD::FMA:
   case ISD::FMAD: {
     if (SNaN)
@@ -6332,7 +6332,8 @@ bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
 }
 
 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
-                                             EVT VT, ArrayRef<SDValue> Ops) {
+                                             EVT VT, ArrayRef<SDValue> Ops,
+                                             SDNodeFlags Flags) {
   // If the opcode is a target-specific ISD node, there's nothing we can
   // do here and the operand rules may not line up with the below, so
   // bail early.
@@ -6550,17 +6551,17 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
 
   ElementCount NumElts = VT.getVectorElementCount();
 
-  // See if we can fold through bitcasted integer ops.
+  // See if we can fold through any bitcasted integer ops.
   if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
       Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
-      Ops[0].getOpcode() == ISD::BITCAST &&
-      Ops[1].getOpcode() == ISD::BITCAST) {
+      (Ops[0].getOpcode() == ISD::BITCAST ||
+       Ops[1].getOpcode() == ISD::BITCAST)) {
     SDValue N1 = peekThroughBitcasts(Ops[0]);
     SDValue N2 = peekThroughBitcasts(Ops[1]);
     auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
     auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
-    EVT BVVT = N1.getValueType();
-    if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
+    if (BV1 && BV2 && N1.getValueType().isInteger() &&
+        N2.getValueType().isInteger()) {
       bool IsLE = getDataLayout().isLittleEndian();
       unsigned EltBits = VT.getScalarSizeInBits();
       SmallVector<APInt> RawBits1, RawBits2;
@@ -6576,15 +6577,22 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
           RawBits.push_back(*Fold);
         }
         if (RawBits.size() == NumElts.getFixedValue()) {
-          // We have constant folded, but we need to cast this again back to
-          // the original (possibly legalized) type.
+          // We have constant folded, but we might need to cast this again back
+          // to the original (possibly legalized) type.
+          EVT BVVT, BVEltVT;
+          if (N1.getValueType() == VT) {
+            BVVT = N1.getValueType();
+            BVEltVT = BV1->getOperand(0).getValueType();
+          } else {
+            BVVT = N2.getValueType();
+            BVEltVT = BV2->getOperand(0).getValueType();
+          }
+          unsigned BVEltBits = BVEltVT.getSizeInBits();
           SmallVector<APInt> DstBits;
           BitVector DstUndefs;
           BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(),
                                            DstBits, RawBits, DstUndefs,
                                            BitVector(RawBits.size(), false));
-          EVT BVEltVT = BV1->getOperand(0).getValueType();
-          unsigned BVEltBits = BVEltVT.getSizeInBits();
           SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
           for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
             if (DstUndefs[I])
@@ -6689,7 +6697,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
     }
 
     // Constant fold the scalar operands.
-    SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps);
+    SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
 
     // Legalize the (integer) scalar constant if necessary.
     if (LegalSVT != SVT)
@@ -7260,7 +7268,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   }
 
   // Perform trivial constant folding.
-  if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
+  if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}, Flags))
     return SV;
 
   // Canonicalize an UNDEF to the RHS, even over a constant.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ba76456b5836a101de398e7906468631269413d6..be5e0f6ef058b018a1c9a66d1146184c047ca5a4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -729,8 +729,17 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
       // prevents it from being picked up by the earlier bitcast case.
       if (ValueVT.getVectorElementCount().isScalar() &&
           (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
-        Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
-                          DAG.getVectorIdxConstant(0, DL));
+        // If we reach this condition and PartVT is FP, this means that
+        // ValueVT is also FP and both have a different size, otherwise we
+        // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
+        // would be invalid since that would mean the smaller FP type has to
+        // be extended to the larger one.
+        if (PartVT.isFloatingPoint()) {
+          Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
+          Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
+        } else
+          Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
+                            DAG.getVectorIdxConstant(0, DL));
       } else {
         uint64_t ValueSize = ValueVT.getFixedSizeInBits();
         assert(PartVT.getFixedSizeInBits() > ValueSize &&
@@ -1684,7 +1693,7 @@ bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
           if (!FragmentExpr)
             continue;
           SDDbgValue *SDV = DAG.getVRegDbgValue(
-              Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
+              Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
           DAG.AddDbgValue(SDV, false);
           Offset += RegisterSize;
         }
@@ -1699,11 +1708,10 @@ bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
   }
 
   // We have created a SDDbgOperand for each Value in Values.
-  // Should use Order instead of SDNodeOrder?
   assert(!LocationOps.empty());
-  SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
-                                        /*IsIndirect=*/false, DbgLoc,
-                                        SDNodeOrder, IsVariadic);
+  SDDbgValue *SDV =
+      DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
+                          /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
   DAG.AddDbgValue(SDV, /*isParameter=*/false);
   return true;
 }
@@ -6742,6 +6750,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::fabs:
   case Intrinsic::sin:
   case Intrinsic::cos:
+  case Intrinsic::tan:
   case Intrinsic::exp10:
   case Intrinsic::floor:
   case Intrinsic::ceil:
@@ -6759,6 +6768,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     case Intrinsic::fabs:         Opcode = ISD::FABS;          break;
     case Intrinsic::sin:          Opcode = ISD::FSIN;          break;
     case Intrinsic::cos:          Opcode = ISD::FCOS;          break;
+    case Intrinsic::tan:          Opcode = ISD::FTAN;          break;
     case Intrinsic::exp10:        Opcode = ISD::FEXP10;        break;
     case Intrinsic::floor:        Opcode = ISD::FFLOOR;        break;
     case Intrinsic::ceil:         Opcode = ISD::FCEIL;         break;
@@ -9160,6 +9170,12 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
         if (visitUnaryFloatCall(I, ISD::FCOS))
           return;
         break;
+      case LibFunc_tan:
+      case LibFunc_tanf:
+      case LibFunc_tanl:
+        if (visitUnaryFloatCall(I, ISD::FTAN))
+          return;
+        break;
       case LibFunc_sqrt:
       case LibFunc_sqrtf:
       case LibFunc_sqrtl:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 2198c2354483ce775a433e96b8a8c1aac25e5883..52da24b594513d86cbef6a29d44d256623e25c85 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -210,6 +210,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FCOS:                       return "fcos";
   case ISD::STRICT_FCOS:                return "strict_fcos";
   case ISD::FSINCOS:                    return "fsincos";
+  case ISD::FTAN:                       return "ftan";
+  case ISD::STRICT_FTAN:                return "strict_ftan";
   case ISD::FTRUNC:                     return "ftrunc";
   case ISD::STRICT_FTRUNC:              return "strict_ftrunc";
   case ISD::FFLOOR:                     return "ffloor";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index f856c8a51984e28123a40e416678223cc52a74f5..c8a5e152576084e44b67c22cae689db87b6c2bf9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1436,11 +1436,9 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                              Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1))
       return true;
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If all of the demanded bits are known one on one side, return the other.
     // These bits cannot contribute to the result of the 'and'.
@@ -1488,7 +1486,7 @@ bool TargetLowering::SimplifyDemandedBits(
       }
       return true;
     }
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
+
     if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1)) {
       if (Flags.hasDisjoint()) {
@@ -1497,7 +1495,6 @@ bool TargetLowering::SimplifyDemandedBits(
       }
       return true;
     }
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If all of the demanded bits are known zero on one side, return the other.
     // These bits cannot contribute to the result of the 'or'.
@@ -1563,11 +1560,9 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                              Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
                              Depth + 1))
       return true;
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If all of the demanded bits are known zero on one side, return the other.
     // These bits cannot contribute to the result of the 'xor'.
@@ -1663,8 +1658,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If the operands are constants, see if we can simplify them.
     if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
@@ -1680,8 +1673,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // Only known if known in both the LHS and RHS.
     Known = Known.intersectWith(Known2);
@@ -1693,8 +1684,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
                              Known2, TLO, Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If the operands are constants, see if we can simplify them.
     if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
@@ -1819,7 +1808,6 @@ bool TargetLowering::SimplifyDemandedBits(
         }
         return true;
       }
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       Known.Zero <<= ShAmt;
       Known.One <<= ShAmt;
       // low bits known zero.
@@ -1993,7 +1981,6 @@ bool TargetLowering::SimplifyDemandedBits(
       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                                Depth + 1))
         return true;
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       Known.Zero.lshrInPlace(ShAmt);
       Known.One.lshrInPlace(ShAmt);
       // High bits known zero.
@@ -2090,7 +2077,6 @@ bool TargetLowering::SimplifyDemandedBits(
       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                                Depth + 1))
         return true;
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       Known.Zero.lshrInPlace(ShAmt);
       Known.One.lshrInPlace(ShAmt);
 
@@ -2385,7 +2371,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
                              Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
 
     // If the sign bit of the input is known set or clear, then we know the
     // top bits of the result.
@@ -2458,7 +2443,6 @@ bool TargetLowering::SimplifyDemandedBits(
       }
       return true;
     }
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     assert(Known.getBitWidth() == InBits && "Src width has changed?");
     Known = Known.zext(BitWidth);
 
@@ -2508,7 +2492,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                              Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     assert(Known.getBitWidth() == InBits && "Src width has changed?");
 
     // If the sign bit is known one, the top bits match.
@@ -2554,7 +2537,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                              Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     assert(Known.getBitWidth() == InBits && "Src width has changed?");
     Known = Known.anyext(BitWidth);
 
@@ -2620,7 +2602,6 @@ bool TargetLowering::SimplifyDemandedBits(
       break;
     }
 
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     break;
   }
   case ISD::AssertZext: {
@@ -2631,7 +2612,6 @@ bool TargetLowering::SimplifyDemandedBits(
     if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
                              TLO, Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
 
     Known.Zero |= ~InMask;
     Known.One &= (~Known.Zero);
@@ -8427,10 +8407,6 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   bool IsMax = Opc == ISD::FMAXIMUM;
   SDNodeFlags Flags = N->getFlags();
 
-  if (VT.isVector() &&
-      isOperationLegalOrCustomOrPromote(Opc, VT.getScalarType()))
-    return SDValue();
-
   // First, implement comparison not propagating NaN. If no native fmin or fmax
   // available, use plain select with setcc instead.
   SDValue MinMax;
@@ -8447,6 +8423,9 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   } else if (isOperationLegalOrCustom(CompOpc, VT)) {
     MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS, Flags);
   } else {
+    if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
+      return DAG.UnrollVectorOp(N);
+
     // NaN (if exists) will be propagated later, so orderness doesn't matter.
     SDValue Compare =
         DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);
@@ -9159,6 +9138,7 @@ SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
   if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
       isOperationLegal(ISD::SMAX, VT)) {
     SDValue Zero = DAG.getConstant(0, dl, VT);
+    Op = DAG.getFreeze(Op);
     return DAG.getNode(ISD::SMAX, dl, VT, Op,
                        DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
   }
@@ -9175,8 +9155,8 @@ SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
   // 0 - abs(x) -> smin(x, sub(0,x))
   if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
       isOperationLegal(ISD::SMIN, VT)) {
-    Op = DAG.getFreeze(Op);
     SDValue Zero = DAG.getConstant(0, dl, VT);
+    Op = DAG.getFreeze(Op);
     return DAG.getNode(ISD::SMIN, dl, VT, Op,
                        DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
   }
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 3aec7049e0cc8343856a4dacd25944eb53649476..8240a1fd7e2ff7ed83ec058907ef730af71f74e2 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -141,6 +141,7 @@ void TargetLoweringBase::InitLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::EXP10_F128, "exp10f128");
     setLibcallName(RTLIB::SIN_F128, "sinf128");
     setLibcallName(RTLIB::COS_F128, "cosf128");
+    setLibcallName(RTLIB::TAN_F128, "tanf128");
     setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
     setLibcallName(RTLIB::POW_F128, "powf128");
     setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
@@ -1015,7 +1016,8 @@ void TargetLoweringBase::initActions() {
   setOperationAction({ISD::FCBRT, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, ISD::FEXP,
                       ISD::FEXP2, ISD::FEXP10, ISD::FFLOOR, ISD::FNEARBYINT,
                       ISD::FCEIL, ISD::FRINT, ISD::FTRUNC, ISD::LROUND,
-                      ISD::LLROUND, ISD::LRINT, ISD::LLRINT, ISD::FROUNDEVEN},
+                      ISD::LLROUND, ISD::LRINT, ISD::LLRINT, ISD::FROUNDEVEN,
+                      ISD::FTAN},
                      {MVT::f32, MVT::f64, MVT::f128}, Expand);
 
   // Default ISD::TRAP to expand (which turns it into abort).
diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index af5d10103f78bec75b078bd9907ed19f90492003..5bf1d265092f6f855f4cf1cc9974fcc75ceb2e26 100644
--- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
@@ -21,7 +22,7 @@ using namespace llvm;
 /// DisableFramePointerElim - This returns true if frame pointer elimination
 /// optimization should be disabled for the given machine function.
 bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
-  // Check to see if the target want to forcably keep frame pointer.
+  // Check to see if the target want to forcibly keep frame pointer.
   if (MF.getSubtarget().getFrameLowering()->keepFramePointer(MF))
     return true;
 
@@ -34,11 +35,27 @@ bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
     return true;
   if (FP == "non-leaf")
     return MF.getFrameInfo().hasCalls();
-  if (FP == "none")
+  if (FP == "none" || FP == "reserved")
     return false;
   llvm_unreachable("unknown frame pointer flag");
 }
 
+bool TargetOptions::FramePointerIsReserved(const MachineFunction &MF) const {
+  // Check to see if the target want to forcibly keep frame pointer.
+  if (MF.getSubtarget().getFrameLowering()->keepFramePointer(MF))
+    return true;
+
+  const Function &F = MF.getFunction();
+
+  if (!F.hasFnAttribute("frame-pointer"))
+    return false;
+
+  StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString();
+  return StringSwitch<bool>(FP)
+      .Cases("all", "non-leaf", "reserved", true)
+      .Case("none", false);
+}
+
 /// HonorSignDependentRoundingFPMath - Return true if the codegen must assume
 /// that the rounding mode of the FPU can change from its default.
 bool TargetOptions::HonorSignDependentRoundingFPMath() const {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index 87a4fc78ceb19c7a5cb34d4074e8196f06e3b2ab..d4979024cb57bdc332974f17cec3a50e9931701b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -240,6 +240,10 @@ static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
                                    ArrayRef<uint64_t> Operands,
                                    unsigned Operand) {
   assert(Operand < Operands.size() && "operand out of bounds");
+  if (!U) {
+    OS << format(" <base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
+    return;
+  }
   auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]);
   if (Die && Die.getTag() == dwarf::DW_TAG_base_type) {
     OS << " (";
@@ -249,8 +253,7 @@ static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
     if (auto Name = dwarf::toString(Die.find(dwarf::DW_AT_name)))
       OS << " \"" << *Name << "\"";
   } else {
-    OS << format(" <invalid base_type ref: 0x%" PRIx64 ">",
-                 Operands[Operand]);
+    OS << format(" <invalid base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 0046220611203c8e136f90a6ffcc9443bedce221..f44a6a472cb6367d87954e6c1011bdf57eeebcb4 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -1183,8 +1183,7 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID,
   StubMap::const_iterator i = Stubs.find(Value);
   if (i != Stubs.end()) {
     resolveRelocation(Section, Offset,
-                      (uint64_t)Section.getAddressWithOffset(i->second),
-                      RelType, 0);
+                      Section.getLoadAddressWithOffset(i->second), RelType, 0);
     LLVM_DEBUG(dbgs() << " Stub function found\n");
   } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) {
     // Create a new stub function.
@@ -1217,8 +1216,7 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID,
       addRelocationForSection(REmovk_g0, Value.SectionID);
     }
     resolveRelocation(Section, Offset,
-                      reinterpret_cast<uint64_t>(Section.getAddressWithOffset(
-                          Section.getStubOffset())),
+                      Section.getLoadAddressWithOffset(Section.getStubOffset()),
                       RelType, 0);
     Section.advanceStubOffset(getMaxStubSize());
   }
@@ -1349,10 +1347,9 @@ RuntimeDyldELF::processRelocationRef(
       // Look for an existing stub.
       StubMap::const_iterator i = Stubs.find(Value);
       if (i != Stubs.end()) {
-        resolveRelocation(
-            Section, Offset,
-            reinterpret_cast<uint64_t>(Section.getAddressWithOffset(i->second)),
-            RelType, 0);
+        resolveRelocation(Section, Offset,
+                          Section.getLoadAddressWithOffset(i->second), RelType,
+                          0);
         LLVM_DEBUG(dbgs() << " Stub function found\n");
       } else {
         // Create a new stub function.
@@ -1367,10 +1364,10 @@ RuntimeDyldELF::processRelocationRef(
         else
           addRelocationForSection(RE, Value.SectionID);
 
-        resolveRelocation(Section, Offset, reinterpret_cast<uint64_t>(
-                                               Section.getAddressWithOffset(
-                                                   Section.getStubOffset())),
-                          RelType, 0);
+        resolveRelocation(
+            Section, Offset,
+            Section.getLoadAddressWithOffset(Section.getStubOffset()), RelType,
+            0);
         Section.advanceStubOffset(getMaxStubSize());
       }
     } else {
@@ -1609,8 +1606,7 @@ RuntimeDyldELF::processRelocationRef(
         if (i != Stubs.end()) {
           // Symbol function stub already created, just relocate to it
           resolveRelocation(Section, Offset,
-                            reinterpret_cast<uint64_t>(
-                                Section.getAddressWithOffset(i->second)),
+                            Section.getLoadAddressWithOffset(i->second),
                             RelType, 0);
           LLVM_DEBUG(dbgs() << " Stub function found\n");
         } else {
@@ -1652,10 +1648,10 @@ RuntimeDyldELF::processRelocationRef(
             addRelocationForSection(REl, Value.SectionID);
           }
 
-          resolveRelocation(Section, Offset, reinterpret_cast<uint64_t>(
-                                                 Section.getAddressWithOffset(
-                                                     Section.getStubOffset())),
-                            RelType, 0);
+          resolveRelocation(
+              Section, Offset,
+              Section.getLoadAddressWithOffset(Section.getStubOffset()),
+              RelType, 0);
           Section.advanceStubOffset(getMaxStubSize());
         }
         if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID)) {
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 2c4b45255d059c81d9c5ff799e4438b1cc657684..92213e19c9d9da2e6a70a99bc5f39ce4ef3ce255 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -3961,7 +3961,7 @@ static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
   UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
 
   // Loop is not unrollable if the loop contains certain instructions.
-  if (!UCE.canUnroll() || UCE.Convergent) {
+  if (!UCE.canUnroll()) {
     LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
     return 1;
   }
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 7a5f18fe2cbd52b87a4dc274dd3187482ab8964b..0bf8be9ac55f9d60ded8fb7d135df4d66cb3aec7 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -326,6 +326,9 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) {
   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
     Out << "aarch64_sme_preservemost_from_x0";
     break;
+  case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
+    Out << "aarch64_sme_preservemost_from_x1";
+    break;
   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
     Out << "aarch64_sme_preservemost_from_x2";
     break;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index a7ed2de6e8a5ff7c56a9fd35d2421d0a8656cc57..2f4b8351e747aa54aeaa4d67aae798eb777a1b1b 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5368,8 +5368,8 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
     return DL.empty() ? std::string("G1") : (DL + "-G1").str();
   }
 
-  if (T.isRISCV64()) {
-    // Make i32 a native type for 64-bit RISC-V.
+  if (T.isLoongArch64() || T.isRISCV64()) {
+    // Make i32 a native type for 64-bit LoongArch and RISC-V.
     auto I = DL.find("-n64-");
     if (I != StringRef::npos)
       return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 29f2cbf611fa3a5a002eac186b321911606662e4..aea9425ebebaab3d4288f789fb52dd9434a63d97 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -181,7 +181,7 @@ template class llvm::SymbolTableListTraits<Instruction,
 BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
                        BasicBlock *InsertBefore)
     : Value(Type::getLabelTy(C), Value::BasicBlockVal),
-      IsNewDbgInfoFormat(false), Parent(nullptr) {
+      IsNewDbgInfoFormat(UseNewDbgInfoFormat), Parent(nullptr) {
 
   if (NewParent)
     insertInto(NewParent, InsertBefore);
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 7b8b9a7b652daec21a7393bff1193c099b44624a..77a833610a3a911695938100ae625c3203f06ecd 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -1416,64 +1416,16 @@ static Constant *foldGEPOfGEP(GEPOperator *GEP, Type *PointeeTy, bool InBounds,
   if (GEP->getInRange())
     return nullptr;
 
+  // Only handle simple case with leading zero index. We cannot perform an
+  // actual addition as we don't know the correct index type size to use.
   Constant *Idx0 = cast<Constant>(Idxs[0]);
-  if (Idx0->isNullValue()) {
-    // Handle the simple case of a zero index.
-    SmallVector<Value*, 16> NewIndices;
-    NewIndices.reserve(Idxs.size() + GEP->getNumIndices());
-    NewIndices.append(GEP->idx_begin(), GEP->idx_end());
-    NewIndices.append(Idxs.begin() + 1, Idxs.end());
-    return ConstantExpr::getGetElementPtr(
-        GEP->getSourceElementType(), cast<Constant>(GEP->getPointerOperand()),
-        NewIndices, InBounds && GEP->isInBounds());
-  }
-
-  gep_type_iterator LastI = gep_type_end(GEP);
-  for (gep_type_iterator I = gep_type_begin(GEP), E = gep_type_end(GEP);
-       I != E; ++I)
-    LastI = I;
-
-  // We can't combine GEPs if the last index is a struct type.
-  if (!LastI.isSequential())
-    return nullptr;
-  // We could perform the transform with non-constant index, but prefer leaving
-  // it as GEP of GEP rather than GEP of add for now.
-  ConstantInt *CI = dyn_cast<ConstantInt>(Idx0);
-  if (!CI)
-    return nullptr;
-
-  // TODO: This code may be extended to handle vectors as well.
-  auto *LastIdx = cast<Constant>(GEP->getOperand(GEP->getNumOperands()-1));
-  Type *LastIdxTy = LastIdx->getType();
-  if (LastIdxTy->isVectorTy())
+  if (!Idx0->isNullValue())
     return nullptr;
 
   SmallVector<Value*, 16> NewIndices;
   NewIndices.reserve(Idxs.size() + GEP->getNumIndices());
-  NewIndices.append(GEP->idx_begin(), GEP->idx_end() - 1);
-
-  // Add the last index of the source with the first index of the new GEP.
-  // Make sure to handle the case when they are actually different types.
-  if (LastIdxTy != Idx0->getType()) {
-    unsigned CommonExtendedWidth =
-        std::max(LastIdxTy->getIntegerBitWidth(),
-                 Idx0->getType()->getIntegerBitWidth());
-    CommonExtendedWidth = std::max(CommonExtendedWidth, 64U);
-
-    Type *CommonTy =
-        Type::getIntNTy(LastIdxTy->getContext(), CommonExtendedWidth);
-    if (Idx0->getType() != CommonTy)
-      Idx0 = ConstantFoldCastInstruction(Instruction::SExt, Idx0, CommonTy);
-    if (LastIdx->getType() != CommonTy)
-      LastIdx =
-          ConstantFoldCastInstruction(Instruction::SExt, LastIdx, CommonTy);
-    if (!Idx0 || !LastIdx)
-      return nullptr;
-  }
-
-  NewIndices.push_back(ConstantExpr::get(Instruction::Add, Idx0, LastIdx));
+  NewIndices.append(GEP->idx_begin(), GEP->idx_end());
   NewIndices.append(Idxs.begin() + 1, Idxs.end());
-
   return ConstantExpr::getGetElementPtr(
       GEP->getSourceElementType(), cast<Constant>(GEP->getPointerOperand()),
       NewIndices, InBounds && GEP->isInBounds());
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index c3bde48b982ced2d3524a422e12956352e52473a..08041c96ffe5a508d1eeaf6314ffd1f05b6ea14f 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -58,8 +58,8 @@ ConstantRange::ConstantRange(APInt L, APInt U)
 
 ConstantRange ConstantRange::fromKnownBits(const KnownBits &Known,
                                            bool IsSigned) {
-  assert(!Known.hasConflict() && "Expected valid KnownBits");
-
+  if (Known.hasConflict())
+    return getEmpty(Known.getBitWidth());
   if (Known.isUnknown())
     return getFull(Known.getBitWidth());
 
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 4a508a022cf7eece7e3ae0142fc0b0cf5e854d28..3a91a4eb6e0d10641036730a3dcb1d4d83189c94 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -3137,16 +3137,35 @@ LLVMBuilderRef LLVMCreateBuilder(void) {
   return LLVMCreateBuilderInContext(LLVMGetGlobalContext());
 }
 
+static void LLVMPositionBuilderImpl(IRBuilder<> *Builder, BasicBlock *Block,
+                                    Instruction *Instr, bool BeforeDbgRecords) {
+  BasicBlock::iterator I = Instr ? Instr->getIterator() : Block->end();
+  I.setHeadBit(BeforeDbgRecords);
+  Builder->SetInsertPoint(Block, I);
+}
+
 void LLVMPositionBuilder(LLVMBuilderRef Builder, LLVMBasicBlockRef Block,
                          LLVMValueRef Instr) {
-  BasicBlock *BB = unwrap(Block);
-  auto I = Instr ? unwrap<Instruction>(Instr)->getIterator() : BB->end();
-  unwrap(Builder)->SetInsertPoint(BB, I);
+  return LLVMPositionBuilderImpl(unwrap(Builder), unwrap(Block),
+                                 unwrap<Instruction>(Instr), false);
+}
+
+void LLVMPositionBuilderBeforeDbgRecords(LLVMBuilderRef Builder,
+                                         LLVMBasicBlockRef Block,
+                                         LLVMValueRef Instr) {
+  return LLVMPositionBuilderImpl(unwrap(Builder), unwrap(Block),
+                                 unwrap<Instruction>(Instr), true);
 }
 
 void LLVMPositionBuilderBefore(LLVMBuilderRef Builder, LLVMValueRef Instr) {
   Instruction *I = unwrap<Instruction>(Instr);
-  unwrap(Builder)->SetInsertPoint(I->getParent(), I->getIterator());
+  return LLVMPositionBuilderImpl(unwrap(Builder), I->getParent(), I, false);
+}
+
+void LLVMPositionBuilderBeforeInstrAndDbgRecords(LLVMBuilderRef Builder,
+                                                 LLVMValueRef Instr) {
+  Instruction *I = unwrap<Instruction>(Instr);
+  return LLVMPositionBuilderImpl(unwrap(Builder), I->getParent(), I, true);
 }
 
 void LLVMPositionBuilderAtEnd(LLVMBuilderRef Builder, LLVMBasicBlockRef Block) {
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 9bd1d7880c9f8131fa619c113dc6d8ed0c55ea92..2b45932093f0fc3cdf887e2491b3597ca663c7b7 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -1404,6 +1404,8 @@ unsigned DIExpression::ExprOperand::getSize() const {
   switch (Op) {
   case dwarf::DW_OP_LLVM_convert:
   case dwarf::DW_OP_LLVM_fragment:
+  case dwarf::DW_OP_LLVM_extract_bits_sext:
+  case dwarf::DW_OP_LLVM_extract_bits_zext:
   case dwarf::DW_OP_bregx:
     return 3;
   case dwarf::DW_OP_constu:
@@ -1474,6 +1476,8 @@ bool DIExpression::isValid() const {
     case dwarf::DW_OP_LLVM_convert:
     case dwarf::DW_OP_LLVM_arg:
     case dwarf::DW_OP_LLVM_tag_offset:
+    case dwarf::DW_OP_LLVM_extract_bits_sext:
+    case dwarf::DW_OP_LLVM_extract_bits_zext:
     case dwarf::DW_OP_constu:
     case dwarf::DW_OP_plus_uconst:
     case dwarf::DW_OP_plus:
diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp
index fbca7cdfcf3f5589297a0d00664e0d8d5b4333ec..9a4926c81dca2776965818af33733beecd061600 100644
--- a/llvm/lib/IR/DebugProgramInstruction.cpp
+++ b/llvm/lib/IR/DebugProgramInstruction.cpp
@@ -366,8 +366,8 @@ void DbgVariableRecord::setKillLocation() {
 }
 
 bool DbgVariableRecord::isKillLocation() const {
-  return (getNumVariableLocationOps() == 0 &&
-          !getExpression()->isComplex()) ||
+  return (!hasArgList() && isa<MDNode>(getRawLocation())) ||
+         (getNumVariableLocationOps() == 0 && !getExpression()->isComplex()) ||
          any_of(location_ops(), [](Value *V) { return isa<UndefValue>(V); });
 }
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 13fa1afeaaff24de52237febb100d8eb2742e1d2..9360e6d7d274c828fba523c838bc3a91b1ad414a 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -83,6 +83,8 @@ static cl::opt<int> NonGlobalValueMaxNameSize(
     "non-global-value-max-name-size", cl::Hidden, cl::init(1024),
     cl::desc("Maximum size for the name of non-global values."));
 
+extern cl::opt<bool> UseNewDbgInfoFormat;
+
 void Function::convertToNewDbgValues() {
   IsNewDbgInfoFormat = true;
   for (auto &BB : *this) {
@@ -383,6 +385,9 @@ Function *Function::createWithDefaultAttr(FunctionType *Ty,
   case FramePointerKind::None:
     // 0 ("none") is the default.
     break;
+  case FramePointerKind::Reserved:
+    B.addAttribute("frame-pointer", "reserved");
+    break;
   case FramePointerKind::NonLeaf:
     B.addAttribute("frame-pointer", "non-leaf");
     break;
@@ -438,7 +443,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace,
     : GlobalObject(Ty, Value::FunctionVal,
                    OperandTraits<Function>::op_begin(this), 0, Linkage, name,
                    computeAddrSpace(AddrSpace, ParentModule)),
-      NumArgs(Ty->getNumParams()), IsNewDbgInfoFormat(false) {
+      NumArgs(Ty->getNumParams()), IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
   assert(FunctionType::isValidReturnType(getReturnType()) &&
          "invalid return type");
   setGlobalObjectSubClassData(0);
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index f97dd18c736c511f4f5ca59fafd77650c73af38b..55c282cb25e79333ac5a594ac892fbe85e657012 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -54,6 +54,8 @@
 
 using namespace llvm;
 
+extern cl::opt<bool> UseNewDbgInfoFormat;
+
 //===----------------------------------------------------------------------===//
 // Methods to implement the globals and functions lists.
 //
@@ -72,7 +74,7 @@ template class llvm::SymbolTableListTraits<GlobalIFunc>;
 Module::Module(StringRef MID, LLVMContext &C)
     : Context(C), ValSymTab(std::make_unique<ValueSymbolTable>(-1)),
       ModuleID(std::string(MID)), SourceFileName(std::string(MID)), DL(""),
-      IsNewDbgInfoFormat(false) {
+      IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
   Context.addModule(this);
 }
 
diff --git a/llvm/lib/IR/PseudoProbe.cpp b/llvm/lib/IR/PseudoProbe.cpp
index df5f78c5118225e5f15317f7f1b458ebb358cbc2..59f218cc3683b2e226b0d52545fd8044f5aa4404 100644
--- a/llvm/lib/IR/PseudoProbe.cpp
+++ b/llvm/lib/IR/PseudoProbe.cpp
@@ -95,13 +95,16 @@ void setProbeDistributionFactor(Instruction &Inst, float Factor) {
             PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
         auto Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes(
             Discriminator);
+        auto DwarfBaseDiscriminator =
+            PseudoProbeDwarfDiscriminator::extractDwarfBaseDiscriminator(
+                Discriminator);
         // Round small factors to 0 to avoid over-counting.
         uint32_t IntFactor =
             PseudoProbeDwarfDiscriminator::FullDistributionFactor;
         if (Factor < 1)
           IntFactor *= Factor;
         uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
-            Index, Type, Attr, IntFactor);
+            Index, Type, Attr, IntFactor, DwarfBaseDiscriminator);
         DIL = DIL->cloneWithDiscriminator(V);
         Inst.setDebugLoc(DIL);
       }
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 684e54444621b538167cac8863192ce39ad428ca..e5927203f33a2087c5e5cb4b0aa9a474f29812e1 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2322,7 +2322,7 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
 
   if (Attrs.hasFnAttr("frame-pointer")) {
     StringRef FP = Attrs.getFnAttr("frame-pointer").getValueAsString();
-    if (FP != "all" && FP != "non-leaf" && FP != "none")
+    if (FP != "all" && FP != "non-leaf" && FP != "none" && FP != "reserved")
       CheckFailed("invalid value for 'frame-pointer' attribute: " + FP, V);
   }
 
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index ad30b5ce9e631442c30ecb69015241602d3394c5..a7157e7a37b4399df45b2959859fc9657b01f6ca 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -66,7 +66,6 @@ STATISTIC(EmittedFillFragments,
 STATISTIC(EmittedNopsFragments, "Number of emitted assembler fragments - nops");
 STATISTIC(EmittedOrgFragments, "Number of emitted assembler fragments - org");
 STATISTIC(evaluateFixup, "Number of evaluated fixups");
-STATISTIC(FragmentLayouts, "Number of fragment layouts");
 STATISTIC(ObjectBytes, "Number of emitted object file bytes");
 STATISTIC(RelaxationSteps, "Number of assembler layout and relaxation steps");
 STATISTIC(RelaxedInstructions, "Number of relaxed instructions");
@@ -404,29 +403,7 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
   llvm_unreachable("invalid fragment kind");
 }
 
-void MCAsmLayout::layoutFragment(MCFragment *F) {
-  MCFragment *Prev = F->getPrevNode();
-
-  // We should never try to recompute something which is valid.
-  assert(!isFragmentValid(F) && "Attempt to recompute a valid fragment!");
-  // We should never try to compute the fragment layout if its predecessor
-  // isn't valid.
-  assert((!Prev || isFragmentValid(Prev)) &&
-         "Attempt to compute fragment before its predecessor!");
-
-  assert(!F->IsBeingLaidOut && "Already being laid out!");
-  F->IsBeingLaidOut = true;
-
-  ++stats::FragmentLayouts;
-
-  // Compute fragment offset and size.
-  if (Prev)
-    F->Offset = Prev->Offset + getAssembler().computeFragmentSize(*this, *Prev);
-  else
-    F->Offset = 0;
-  F->IsBeingLaidOut = false;
-  LastValidFragment[F->getParent()] = F;
-
+void MCAsmLayout::layoutBundle(MCFragment *F) {
   // If bundling is enabled and this fragment has instructions in it, it has to
   // obey the bundling restrictions. With padding, we'll have:
   //
@@ -454,21 +431,40 @@ void MCAsmLayout::layoutFragment(MCFragment *F) {
   // within-fragment padding (which would produce less padding when N is less
   // than the bundle size), but for now we don't.
   //
-  if (Assembler.isBundlingEnabled() && F->hasInstructions()) {
-    assert(isa<MCEncodedFragment>(F) &&
-           "Only MCEncodedFragment implementations have instructions");
-    MCEncodedFragment *EF = cast<MCEncodedFragment>(F);
-    uint64_t FSize = Assembler.computeFragmentSize(*this, *EF);
-
-    if (!Assembler.getRelaxAll() && FSize > Assembler.getBundleAlignSize())
-      report_fatal_error("Fragment can't be larger than a bundle size");
-
-    uint64_t RequiredBundlePadding =
-        computeBundlePadding(Assembler, EF, EF->Offset, FSize);
-    if (RequiredBundlePadding > UINT8_MAX)
-      report_fatal_error("Padding cannot exceed 255 bytes");
-    EF->setBundlePadding(static_cast<uint8_t>(RequiredBundlePadding));
-    EF->Offset += RequiredBundlePadding;
+  assert(isa<MCEncodedFragment>(F) &&
+         "Only MCEncodedFragment implementations have instructions");
+  MCEncodedFragment *EF = cast<MCEncodedFragment>(F);
+  uint64_t FSize = Assembler.computeFragmentSize(*this, *EF);
+
+  if (!Assembler.getRelaxAll() && FSize > Assembler.getBundleAlignSize())
+    report_fatal_error("Fragment can't be larger than a bundle size");
+
+  uint64_t RequiredBundlePadding =
+      computeBundlePadding(Assembler, EF, EF->Offset, FSize);
+  if (RequiredBundlePadding > UINT8_MAX)
+    report_fatal_error("Padding cannot exceed 255 bytes");
+  EF->setBundlePadding(static_cast<uint8_t>(RequiredBundlePadding));
+  EF->Offset += RequiredBundlePadding;
+}
+
+uint64_t MCAsmLayout::getFragmentOffset(const MCFragment *F) const {
+  ensureValid(F);
+  return F->Offset;
+}
+
+void MCAsmLayout::ensureValid(const MCFragment *Frag) const {
+  MCSection &Sec = *Frag->getParent();
+  if (Sec.hasLayout())
+    return;
+  Sec.setHasLayout(true);
+  uint64_t Offset = 0;
+  for (MCFragment &F : Sec) {
+    F.Offset = Offset;
+    if (Assembler.isBundlingEnabled() && F.hasInstructions()) {
+      const_cast<MCAsmLayout *>(this)->layoutBundle(&F);
+      Offset = F.Offset;
+    }
+    Offset += getAssembler().computeFragmentSize(*this, F);
   }
 }
 
@@ -848,7 +844,7 @@ void MCAssembler::layout(MCAsmLayout &Layout) {
     // another. If any fragment has changed size, we have to re-layout (and
     // as a result possibly further relax) all.
     for (MCSection &Sec : *this)
-      Layout.invalidateFragmentsFrom(&*Sec.begin());
+      Sec.setHasLayout(false);
   }
 
   DEBUG_WITH_TYPE("mc-dump", {
@@ -1098,9 +1094,11 @@ bool MCAssembler::relaxBoundaryAlign(MCAsmLayout &Layout,
 
   uint64_t AlignedOffset = Layout.getFragmentOffset(&BF);
   uint64_t AlignedSize = 0;
-  for (const MCFragment *F = BF.getLastFragment(); F != &BF;
-       F = F->getPrevNode())
+  for (const MCFragment *F = BF.getNextNode();; F = F->getNextNode()) {
     AlignedSize += computeFragmentSize(Layout, *F);
+    if (F == BF.getLastFragment())
+      break;
+  }
 
   Align BoundaryAlignment = BF.getAlignment();
   uint64_t NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
@@ -1109,7 +1107,6 @@ bool MCAssembler::relaxBoundaryAlign(MCAsmLayout &Layout,
   if (NewSize == BF.getSize())
     return false;
   BF.setSize(NewSize);
-  Layout.invalidateFragmentsFrom(&BF);
   return true;
 }
 
@@ -1219,47 +1216,19 @@ bool MCAssembler::relaxFragment(MCAsmLayout &Layout, MCFragment &F) {
   }
 }
 
-bool MCAssembler::layoutSectionOnce(MCAsmLayout &Layout, MCSection &Sec) {
-  // Holds the first fragment which needed relaxing during this layout. It will
-  // remain NULL if none were relaxed.
-  // When a fragment is relaxed, all the fragments following it should get
-  // invalidated because their offset is going to change.
-  MCFragment *FirstRelaxedFragment = nullptr;
-
-  // Attempt to relax all the fragments in the section.
-  for (MCFragment &Frag : Sec) {
-    // Check if this is a fragment that needs relaxation.
-    bool RelaxedFrag = relaxFragment(Layout, Frag);
-    if (RelaxedFrag && !FirstRelaxedFragment)
-      FirstRelaxedFragment = &Frag;
-  }
-  if (FirstRelaxedFragment) {
-    Layout.invalidateFragmentsFrom(FirstRelaxedFragment);
-    return true;
-  }
-  return false;
-}
-
 bool MCAssembler::layoutOnce(MCAsmLayout &Layout) {
   ++stats::RelaxationSteps;
 
-  bool WasRelaxed = false;
-  for (MCSection &Sec : *this) {
-    while (layoutSectionOnce(Layout, Sec))
-      WasRelaxed = true;
-  }
-
-  return WasRelaxed;
+  bool Changed = false;
+  for (MCSection &Sec : *this)
+    for (MCFragment &Frag : Sec)
+      if (relaxFragment(Layout, Frag))
+        Changed = true;
+  return Changed;
 }
 
 void MCAssembler::finishLayout(MCAsmLayout &Layout) {
   assert(getBackendPtr() && "Expected assembler backend");
-  // The layout is done. Mark every fragment as valid.
-  for (unsigned int i = 0, n = Layout.getSectionOrder().size(); i != n; ++i) {
-    MCSection &Section = *Layout.getSectionOrder()[i];
-    Layout.getFragmentOffset(&*Section.getFragmentList().rbegin());
-    computeFragmentSize(Layout, *Section.getFragmentList().rbegin());
-  }
   getBackend().finishLayout(*this, Layout);
 }
 
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index f027eb65d700e8aef20dbfef2af17d836aa77e5e..771ca9c6006ca0c6d3521386e7bbf13edb3868ea 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -498,7 +498,7 @@ MCSectionELF *MCContext::createELFSectionImpl(StringRef Section, unsigned Type,
                    R, LinkedToSym);
 
   auto *F = new MCDataFragment();
-  Ret->getFragmentList().insert(Ret->begin(), F);
+  Ret->addFragment(*F);
   F->setParent(Ret);
   R->setFragment(F);
 
@@ -772,7 +772,7 @@ MCSectionWasm *MCContext::getWasmSection(const Twine &Section, SectionKind Kind,
   Entry.second = Result;
 
   auto *F = new MCDataFragment();
-  Result->getFragmentList().insert(Result->begin(), F);
+  Result->addFragment(*F);
   F->setParent(Result);
   Begin->setFragment(F);
 
@@ -838,7 +838,7 @@ MCSectionXCOFF *MCContext::getXCOFFSection(
   Entry.second = Result;
 
   auto *F = new MCDataFragment();
-  Result->getFragmentList().insert(Result->begin(), F);
+  Result->addFragment(*F);
   F->setParent(Result);
 
   if (Begin)
@@ -861,7 +861,7 @@ MCSectionSPIRV *MCContext::getSPIRVSection() {
       MCSectionSPIRV(SectionKind::getText(), Begin);
 
   auto *F = new MCDataFragment();
-  Result->getFragmentList().insert(Result->begin(), F);
+  Result->addFragment(*F);
   F->setParent(Result);
 
   return Result;
@@ -884,7 +884,7 @@ MCSectionDXContainer *MCContext::getDXContainerSection(StringRef Section,
 
   // The first fragment will store the header
   auto *F = new MCDataFragment();
-  MapIt->second->getFragmentList().insert(MapIt->second->begin(), F);
+  MapIt->second->addFragment(*F);
   F->setParent(MapIt->second);
 
   return MapIt->second;
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index b065d03651c4527163439debfde17d7ca8412702..b70ac86c18ccf16e2e714888efa1208c48b2ede4 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -645,10 +645,6 @@ static void AttemptToFoldSymbolOffsetDifference(
       Addend += SA.getOffset() - SB.getOffset();
       return FinalizeFolding();
     }
-    // One of the symbol involved is part of a fragment being laid out. Quit now
-    // to avoid a self loop.
-    if (!Layout->canGetFragmentOffset(FA) || !Layout->canGetFragmentOffset(FB))
-      return;
 
     // Eagerly evaluate when layout is finalized.
     Addend += Layout->getSymbolOffset(A->getSymbol()) -
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index a8da46dbd872795e587f6869de5db7659aa9cce2..84a587164c788d213a06f6edaca36271af4d490a 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -39,64 +39,8 @@ MCAsmLayout::MCAsmLayout(MCAssembler &Asm) : Assembler(Asm) {
       SectionOrder.push_back(&Sec);
 }
 
-bool MCAsmLayout::isFragmentValid(const MCFragment *F) const {
-  const MCSection *Sec = F->getParent();
-  const MCFragment *LastValid = LastValidFragment.lookup(Sec);
-  if (!LastValid)
-    return false;
-  assert(LastValid->getParent() == Sec);
-  return F->getLayoutOrder() <= LastValid->getLayoutOrder();
-}
-
-bool MCAsmLayout::canGetFragmentOffset(const MCFragment *F) const {
-  MCSection *Sec = F->getParent();
-  MCSection::iterator I;
-  if (MCFragment *LastValid = LastValidFragment[Sec]) {
-    // Fragment already valid, offset is available.
-    if (F->getLayoutOrder() <= LastValid->getLayoutOrder())
-      return true;
-    I = ++MCSection::iterator(LastValid);
-  } else
-    I = Sec->begin();
-
-  // A fragment ordered before F is currently being laid out.
-  const MCFragment *FirstInvalidFragment = &*I;
-  if (FirstInvalidFragment->IsBeingLaidOut)
-    return false;
-
-  return true;
-}
-
 void MCAsmLayout::invalidateFragmentsFrom(MCFragment *F) {
-  // If this fragment wasn't already valid, we don't need to do anything.
-  if (!isFragmentValid(F))
-    return;
-
-  // Otherwise, reset the last valid fragment to the previous fragment
-  // (if this is the first fragment, it will be NULL).
-  LastValidFragment[F->getParent()] = F->getPrevNode();
-}
-
-void MCAsmLayout::ensureValid(const MCFragment *F) const {
-  MCSection *Sec = F->getParent();
-  MCSection::iterator I;
-  if (MCFragment *Cur = LastValidFragment[Sec])
-    I = ++MCSection::iterator(Cur);
-  else
-    I = Sec->begin();
-
-  // Advance the layout position until the fragment is valid.
-  while (!isFragmentValid(F)) {
-    assert(I != Sec->end() && "Layout bookkeeping error");
-    const_cast<MCAsmLayout *>(this)->layoutFragment(&*I);
-    ++I;
-  }
-}
-
-uint64_t MCAsmLayout::getFragmentOffset(const MCFragment *F) const {
-  ensureValid(F);
-  assert(F->Offset != ~UINT64_C(0) && "Address not set!");
-  return F->Offset;
+  F->getParent()->setHasLayout(false);
 }
 
 // Simple getSymbolOffset helper for the non-variable case.
@@ -258,9 +202,9 @@ void ilist_alloc_traits<MCFragment>::deleteNode(MCFragment *V) { V->destroy(); }
 MCFragment::MCFragment(FragmentType Kind, bool HasInstructions,
                        MCSection *Parent)
     : Parent(Parent), Atom(nullptr), Offset(~UINT64_C(0)), LayoutOrder(0),
-      Kind(Kind), IsBeingLaidOut(false), HasInstructions(HasInstructions) {
+      Kind(Kind), HasInstructions(HasInstructions) {
   if (Parent && !isa<MCDummyFragment>(*this))
-    Parent->getFragmentList().push_back(this);
+    Parent->addFragment(*this);
 }
 
 void MCFragment::destroy() {
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 0fb9e8e13910f80c6760550ccc6d50d37398c6d5..12e69f70537b78c06197b0e67aefad1577e44bf4 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -23,8 +23,8 @@ using namespace llvm;
 MCSection::MCSection(SectionVariant V, StringRef Name, SectionKind K,
                      MCSymbol *Begin)
     : Begin(Begin), BundleGroupBeforeFirstInst(false), HasInstructions(false),
-      IsRegistered(false), DummyFragment(this), Name(Name), Variant(V),
-      Kind(K) {}
+      HasLayout(false), IsRegistered(false), DummyFragment(this), Name(Name),
+      Variant(V), Kind(K) {}
 
 MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) {
   if (!End)
@@ -128,7 +128,7 @@ void MCSection::flushPendingLabels() {
 LLVM_DUMP_METHOD void MCSection::dump() const {
   raw_ostream &OS = errs();
 
-  OS << "<MCSection";
+  OS << "<MCSection Name:" << getName();
   OS << " Fragments:[\n      ";
   for (auto it = begin(), ie = end(); it != ie; ++it) {
     if (it != begin())
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 985f9351f4a3085fffb14a6689f9b280b3bdf6a2..788e92f94b2689ea67c45c87f9348bdc4625a309 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -877,7 +877,7 @@ void WasmObjectWriter::writeImportSection(ArrayRef<wasm::WasmImport> Imports,
       break;
     case wasm::WASM_EXTERNAL_TABLE:
       W->OS << char(Import.Table.ElemType);
-      encodeULEB128(0, W->OS);           // flags
+      encodeULEB128(Import.Table.Limits.Flags, W->OS);
       encodeULEB128(NumElements, W->OS); // initial
       break;
     case wasm::WASM_EXTERNAL_TAG:
@@ -1022,7 +1022,8 @@ void WasmObjectWriter::writeElemSection(
     encodeULEB128(TableNumber, W->OS); // the table number
 
   // init expr for starting offset
-  W->OS << char(wasm::WASM_OPCODE_I32_CONST);
+  W->OS << char(is64Bit() ? wasm::WASM_OPCODE_I64_CONST
+                          : wasm::WASM_OPCODE_I32_CONST);
   encodeSLEB128(InitialTableOffset, W->OS);
   W->OS << char(wasm::WASM_OPCODE_END);
 
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index 2b6bdbf24afa073c0f0fa85f5a2de866ed716949..cbc55a145e0e6f1e86852a602d1896b627bc5066 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -586,6 +586,8 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const {
     return "gfx1150";
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151:
     return "gfx1151";
+  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152:
+    return "gfx1152";
 
   // AMDGCN GFX12.
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200:
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 8e2a9481c922703a4af742e22ee8c1ddc5515f3d..0fee299994bc450ac9425bbc78233ab21dcb075c 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -611,6 +611,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1103, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1150, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1151, EF_AMDGPU_MACH);
+    BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1152, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1200, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1201, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, EF_AMDGPU_MACH);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 316d05bf1dc371a640004aa6ee3517400e2268f2..19e8a8ab68a73d4a2a017d5ba3a46ad922262e86 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -97,6 +97,7 @@
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/RegAllocFast.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
 #include "llvm/CodeGen/ShadowStackGCLowering.h"
@@ -139,6 +140,7 @@
 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
 #include "llvm/Transforms/IPO/EmbedBitcodePass.h"
+#include "llvm/Transforms/IPO/ExpandVariadics.h"
 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
 #include "llvm/Transforms/IPO/FunctionAttrs.h"
 #include "llvm/Transforms/IPO/FunctionImport.h"
@@ -297,6 +299,7 @@
 #include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h"
 #include "llvm/Transforms/Utils/UnifyLoopExits.h"
 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
+#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
 #include "llvm/Transforms/Vectorize/VectorCombine.h"
@@ -1162,6 +1165,38 @@ Expected<SmallVector<std::string, 0>> parseInternalizeGVs(StringRef Params) {
   return Expected<SmallVector<std::string, 0>>(std::move(PreservedGVs));
 }
 
+Expected<RegAllocFastPassOptions>
+parseRegAllocFastPassOptions(PassBuilder &PB, StringRef Params) {
+  RegAllocFastPassOptions Opts;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName.consume_front("filter=")) {
+      RegClassFilterFunc Filter = PB.parseRegAllocFilter(ParamName);
+      if (!Filter) {
+        return make_error<StringError>(
+            formatv("invalid regallocfast register filter '{0}' ", ParamName)
+                .str(),
+            inconvertibleErrorCode());
+      }
+      Opts.Filter = Filter;
+      Opts.FilterName = ParamName;
+      continue;
+    }
+
+    if (ParamName == "no-clear-vregs") {
+      Opts.ClearVRegs = false;
+      continue;
+    }
+
+    return make_error<StringError>(
+        formatv("invalid regallocfast pass parameter '{0}' ", ParamName).str(),
+        inconvertibleErrorCode());
+  }
+  return Opts;
+}
+
 } // namespace
 
 /// Tests whether a pass name starts with a valid prefix for a default pipeline
@@ -1295,6 +1330,11 @@ static bool isMachineFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)                               \
   if (Name == NAME)                                                            \
     return true;
+#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER,    \
+                                          PARAMS)                              \
+  if (PassBuilder::checkParametrizedPassName(Name, NAME))                      \
+    return true;
+
 #define MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS)                           \
   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
     return true;
@@ -1924,6 +1964,15 @@ Error PassBuilder::parseMachinePass(MachineFunctionPassManager &MFPM,
     MFPM.addPass(CREATE_PASS);                                                 \
     return Error::success();                                                   \
   }
+#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER,    \
+                                          PARAMS)                              \
+  if (checkParametrizedPassName(Name, NAME)) {                                 \
+    auto Params = parsePassParameters(PARSER, Name, NAME);                     \
+    if (!Params)                                                               \
+      return Params.takeError();                                               \
+    MFPM.addPass(CREATE_PASS(Params.get()));                                   \
+    return Error::success();                                                   \
+  }
 #include "llvm/Passes/MachinePassRegistry.def"
 
   for (auto &C : MachineFunctionPipelineParsingCallbacks)
@@ -2171,6 +2220,15 @@ Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
   return Error::success();
 }
 
+RegClassFilterFunc PassBuilder::parseRegAllocFilter(StringRef FilterName) {
+  if (FilterName == "all")
+    return allocateAllRegClasses;
+  for (auto &C : RegClassFilterParsingCallbacks)
+    if (auto F = C(FilterName))
+      return F;
+  return nullptr;
+}
+
 static void printPassName(StringRef PassName, raw_ostream &OS) {
   OS << "  " << PassName << "\n";
 }
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 50682ca4970f195fa18fec902707400b5482d40f..60c517790bcab6e03a0307fa4b79766e7ec2b758 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -59,6 +59,7 @@ MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass())
 MODULE_PASS("dxil-upgrade", DXILUpgradePass())
 MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass())
 MODULE_PASS("extract-blocks", BlockExtractorPass({}, false))
+MODULE_PASS("expand-variadics", ExpandVariadicsPass(ExpandVariadicsMode::Disable))
 MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("globalopt", GlobalOptPass())
@@ -621,6 +622,7 @@ LOOP_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 LOOP_PASS("loop-bound-split", LoopBoundSplitPass())
 LOOP_PASS("loop-deletion", LoopDeletionPass())
 LOOP_PASS("loop-idiom", LoopIdiomRecognizePass())
+LOOP_PASS("loop-idiom-vectorize", LoopIdiomVectorizePass())
 LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass())
 LOOP_PASS("loop-predication", LoopPredicationPass())
 LOOP_PASS("loop-reduce", LoopStrengthReducePass())
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index c7adc7668b9a176c5c8ecd817f2fc12d0d7a8802..b23d5fe245481c86ca15973e8ece2fae795841f6 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1357,7 +1357,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
   bool Registered = false;
   PIC.registerBeforeNonSkippedPassCallback([this, &MAM, Registered](
                                                StringRef P, Any IR) mutable {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     assert(&PassStack.emplace_back(P));
 #endif
     (void)this;
@@ -1386,7 +1386,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 
   PIC.registerAfterPassInvalidatedCallback(
       [this](StringRef P, const PreservedAnalyses &PassPA) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
         assert(PassStack.pop_back_val() == P &&
                "Before and After callbacks must correspond");
 #endif
@@ -1395,7 +1395,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 
   PIC.registerAfterPassCallback([this, &MAM](StringRef P, Any IR,
                                              const PreservedAnalyses &PassPA) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     assert(PassStack.pop_back_val() == P &&
            "Before and After callbacks must correspond");
 #endif
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index ad63086430bf6d39be942e90d1d259c3f30df5b3..d707c07e6dc5087e89cf86545b6edf873050c3cc 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -282,7 +282,7 @@ std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage,
 static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
   uint32_t Count = NumPrefix;
   uint32_t Pos = 0, LastPos = 0;
-  for (auto & CI : PathNameStr) {
+  for (const auto &CI : PathNameStr) {
     ++Pos;
     if (llvm::sys::path::is_separator(CI)) {
       LastPos = Pos;
@@ -1176,16 +1176,6 @@ void ValueProfData::deserializeTo(InstrProfRecord &Record,
   }
 }
 
-template <class T>
-static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) {
-  using namespace support;
-
-  if (Orig == llvm::endianness::little)
-    return endian::readNext<T, llvm::endianness::little>(D);
-  else
-    return endian::readNext<T, llvm::endianness::big>(D);
-}
-
 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
   return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
                                             ValueProfData());
@@ -1224,7 +1214,8 @@ ValueProfData::getValueProfData(const unsigned char *D,
     return make_error<InstrProfError>(instrprof_error::truncated);
 
   const unsigned char *Header = D;
-  uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
+  uint32_t TotalSize = endian::readNext<uint32_t>(Header, Endianness);
+
   if (D + TotalSize > BufferEnd)
     return make_error<InstrProfError>(instrprof_error::too_large);
 
@@ -1308,7 +1299,7 @@ void annotateValueSite(Module &M, Instruction &Inst,
 
   // Value Profile Data
   uint32_t MDCount = MaxMDCount;
-  for (auto &VD : VDs) {
+  for (const auto &VD : VDs) {
     Vals.push_back(MDHelper.createConstant(
         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
     Vals.push_back(MDHelper.createConstant(
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 8298d3d3729511a3ce1730578d478415dd9fcf47..e18ce5d373d1cb75c098b495fac80cc314133204 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -91,12 +91,14 @@ static Error initializeReader(InstrProfReader &Reader) {
 /// associated endian format to read the binary ids correctly.
 static Error
 readBinaryIdsInternal(const MemoryBuffer &DataBuffer,
-                      const uint64_t BinaryIdsSize,
-                      const uint8_t *BinaryIdsStart,
+                      ArrayRef<uint8_t> BinaryIdsBuffer,
                       std::vector<llvm::object::BuildID> &BinaryIds,
                       const llvm::endianness Endian) {
   using namespace support;
 
+  const uint64_t BinaryIdsSize = BinaryIdsBuffer.size();
+  const uint8_t *BinaryIdsStart = BinaryIdsBuffer.data();
+
   if (BinaryIdsSize == 0)
     return Error::success();
 
@@ -113,12 +115,7 @@ readBinaryIdsInternal(const MemoryBuffer &DataBuffer,
           instrprof_error::malformed,
           "not enough data to read binary id length");
 
-    uint64_t BILen = 0;
-    if (Endian == llvm::endianness::little)
-      BILen = endian::readNext<uint64_t, llvm::endianness::little>(BI);
-    else
-      BILen = endian::readNext<uint64_t, llvm::endianness::big>(BI);
-
+    uint64_t BILen = endian::readNext<uint64_t>(BI, Endian);
     if (BILen == 0)
       return make_error<InstrProfError>(instrprof_error::malformed,
                                         "binary id length is 0");
@@ -143,13 +140,12 @@ readBinaryIdsInternal(const MemoryBuffer &DataBuffer,
   return Error::success();
 }
 
-static void
-printBinaryIdsInternal(raw_ostream &OS,
-                       std::vector<llvm::object::BuildID> &BinaryIds) {
+static void printBinaryIdsInternal(raw_ostream &OS,
+                                   ArrayRef<llvm::object::BuildID> BinaryIds) {
   OS << "Binary IDs: \n";
-  for (auto BI : BinaryIds) {
-    for (uint64_t I = 0; I < BI.size(); I++)
-      OS << format("%02x", BI[I]);
+  for (const auto &BI : BinaryIds) {
+    for (auto I : BI)
+      OS << format("%02x", I);
     OS << "\n";
   }
 }
@@ -590,10 +586,10 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
   if (BinaryIdSize % sizeof(uint64_t) || BinaryIdEnd > BufferEnd)
     return error(instrprof_error::bad_header);
-  if (BinaryIdSize != 0) {
-    if (Error Err =
-            readBinaryIdsInternal(*DataBuffer, BinaryIdSize, BinaryIdStart,
-                                  BinaryIds, getDataEndianness()))
+  ArrayRef<uint8_t> BinaryIdsBuffer(BinaryIdStart, BinaryIdSize);
+  if (!BinaryIdsBuffer.empty()) {
+    if (Error Err = readBinaryIdsInternal(*DataBuffer, BinaryIdsBuffer,
+                                          BinaryIds, getDataEndianness()))
       return Err;
   }
 
@@ -1204,8 +1200,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
 
 Error IndexedMemProfReader::deserializeV012(const unsigned char *Start,
                                             const unsigned char *Ptr,
-                                            uint64_t FirstWord,
-                                            memprof::IndexedVersion Version) {
+                                            uint64_t FirstWord) {
   // The value returned from RecordTableGenerator.Emit.
   const uint64_t RecordTableOffset =
       Version == memprof::Version0
@@ -1259,8 +1254,7 @@ Error IndexedMemProfReader::deserializeV012(const unsigned char *Start,
 }
 
 Error IndexedMemProfReader::deserializeV3(const unsigned char *Start,
-                                          const unsigned char *Ptr,
-                                          memprof::IndexedVersion Version) {
+                                          const unsigned char *Ptr) {
   // The offset in the stream right before invoking
   // CallStackTableGenerator.Emit.
   const uint64_t CallStackPayloadOffset =
@@ -1285,7 +1279,7 @@ Error IndexedMemProfReader::deserializeV3(const unsigned char *Start,
   MemProfRecordTable.reset(MemProfRecordHashTable::Create(
       /*Buckets=*/Start + RecordTableOffset,
       /*Payload=*/Start + RecordPayloadOffset,
-      /*Base=*/Start, memprof::RecordLookupTrait(Version, Schema)));
+      /*Base=*/Start, memprof::RecordLookupTrait(memprof::Version3, Schema)));
 
   return Error::success();
 }
@@ -1323,11 +1317,11 @@ Error IndexedMemProfReader::deserialize(const unsigned char *Start,
   case memprof::Version0:
   case memprof::Version1:
   case memprof::Version2:
-    if (Error E = deserializeV012(Start, Ptr, FirstWord, Version))
+    if (Error E = deserializeV012(Start, Ptr, FirstWord))
       return E;
     break;
   case memprof::Version3:
-    if (Error E = deserializeV3(Start, Ptr, Version))
+    if (Error E = deserializeV3(Start, Ptr))
       return E;
     break;
   }
@@ -1391,13 +1385,13 @@ Error IndexedInstrProfReader::readHeader() {
   if (Header->getIndexedProfileVersion() >= 9) {
     const unsigned char *Ptr = Start + Header->BinaryIdOffset;
     // Read binary ids size.
-    BinaryIdsSize =
+    uint64_t BinaryIdsSize =
         support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
     if (BinaryIdsSize % sizeof(uint64_t))
       return error(instrprof_error::bad_header);
     // Set the binary ids start.
-    BinaryIdsStart = Ptr;
-    if (BinaryIdsStart > (const unsigned char *)DataBuffer->getBufferEnd())
+    BinaryIdsBuffer = ArrayRef<uint8_t>(Ptr, BinaryIdsSize);
+    if (Ptr > (const unsigned char *)DataBuffer->getBufferEnd())
       return make_error<InstrProfError>(instrprof_error::malformed,
                                         "corrupted binary ids");
   }
@@ -1405,14 +1399,16 @@ Error IndexedInstrProfReader::readHeader() {
   if (Header->getIndexedProfileVersion() >= 12) {
     const unsigned char *Ptr = Start + Header->VTableNamesOffset;
 
-    CompressedVTableNamesLen =
+    uint64_t CompressedVTableNamesLen =
         support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
 
     // Writer first writes the length of compressed string, and then the actual
     // content.
-    VTableNamePtr = (const char *)Ptr;
+    const char *VTableNamePtr = (const char *)Ptr;
     if (VTableNamePtr > (const char *)DataBuffer->getBufferEnd())
       return make_error<InstrProfError>(instrprof_error::truncated);
+
+    VTableName = StringRef(VTableNamePtr, CompressedVTableNamesLen);
   }
 
   if (Header->getIndexedProfileVersion() >= 10 &&
@@ -1468,8 +1464,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
 
   auto NewSymtab = std::make_unique<InstrProfSymtab>();
 
-  if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
-          StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
+  if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(VTableName)) {
     auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
     consumeError(error(ErrCode, Msg));
   }
@@ -1509,7 +1504,7 @@ Expected<InstrProfRecord> IndexedInstrProfReader::getInstrProfRecord(
   // A flag to indicate if the records are from the same type
   // of profile (i.e cs vs nocs).
   bool CSBitMatch = false;
-  auto getFuncSum = [](const std::vector<uint64_t> &Counts) {
+  auto getFuncSum = [](ArrayRef<uint64_t> Counts) {
     uint64_t ValueSum = 0;
     for (uint64_t CountValue : Counts) {
       if (CountValue == (uint64_t)-1)
@@ -1698,8 +1693,8 @@ Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
 
 Error IndexedInstrProfReader::readBinaryIds(
     std::vector<llvm::object::BuildID> &BinaryIds) {
-  return readBinaryIdsInternal(*DataBuffer, BinaryIdsSize, BinaryIdsStart,
-                               BinaryIds, llvm::endianness::little);
+  return readBinaryIdsInternal(*DataBuffer, BinaryIdsBuffer, BinaryIds,
+                               llvm::endianness::little);
 }
 
 Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index e58e6b8acfc81269463adfb17b75576b8d1abe83..1a9add109a360a0469d794fafcbab035c9e92448 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -55,7 +55,7 @@ public:
   ProfOStream(raw_string_ostream &STR)
       : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
 
-  uint64_t tell() { return OS.tell(); }
+  [[nodiscard]] uint64_t tell() const { return OS.tell(); }
   void write(uint64_t V) { LE.write<uint64_t>(V); }
   void write32(uint32_t V) { LE.write<uint32_t>(V); }
   void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
@@ -494,17 +494,40 @@ static uint64_t writeMemProfFrames(
 static llvm::DenseMap<memprof::FrameId, memprof::LinearFrameId>
 writeMemProfFrameArray(
     ProfOStream &OS,
-    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
+    llvm::DenseMap<memprof::FrameId, memprof::FrameStat> &FrameHistogram) {
   // Mappings from FrameIds to array indexes.
   llvm::DenseMap<memprof::FrameId, memprof::LinearFrameId> MemProfFrameIndexes;
 
-  // Sort the FrameIDs for stability.
+  // Compute the order in which we serialize Frames.  The order does not matter
+  // in terms of correctness, but we still compute it for deserialization
+  // performance.  Specifically, if we serialize frequently used Frames one
+  // after another, we have better cache utilization.  For two Frames that
+  // appear equally frequently, we break a tie by serializing the one that tends
+  // to appear earlier in call stacks.  We implement the tie-breaking mechanism
+  // by computing the sum of indexes within call stacks for each Frame.  If we
+  // still have a tie, then we just resort to compare two FrameIds, which is
+  // just for stability of output.
   std::vector<std::pair<memprof::FrameId, const memprof::Frame *>> FrameIdOrder;
   FrameIdOrder.reserve(MemProfFrameData.size());
   for (const auto &[Id, Frame] : MemProfFrameData)
     FrameIdOrder.emplace_back(Id, &Frame);
   assert(MemProfFrameData.size() == FrameIdOrder.size());
-  llvm::sort(FrameIdOrder);
+  llvm::sort(FrameIdOrder,
+             [&](const std::pair<memprof::FrameId, const memprof::Frame *> &L,
+                 const std::pair<memprof::FrameId, const memprof::Frame *> &R) {
+               const auto &SL = FrameHistogram[L.first];
+               const auto &SR = FrameHistogram[R.first];
+               // Popular FrameIds should come first.
+               if (SL.Count != SR.Count)
+                 return SL.Count > SR.Count;
+               // If they are equally popular, then the one that tends to appear
+               // earlier in call stacks should come first.
+               if (SL.PositionSum != SR.PositionSum)
+                 return SL.PositionSum < SR.PositionSum;
+               // Compare their FrameIds for sort stability.
+               return L.first < R.first;
+             });
 
   // Serialize all frames while creating mappings from linear IDs to FrameIds.
   uint64_t Index = 0;
@@ -543,23 +566,17 @@ writeMemProfCallStackArray(
     llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
         &MemProfCallStackData,
     llvm::DenseMap<memprof::FrameId, memprof::LinearFrameId>
-        &MemProfFrameIndexes) {
+        &MemProfFrameIndexes,
+    llvm::DenseMap<memprof::FrameId, memprof::FrameStat> &FrameHistogram) {
   llvm::DenseMap<memprof::CallStackId, memprof::LinearCallStackId>
       MemProfCallStackIndexes;
 
-  MemProfCallStackIndexes.reserve(MemProfCallStackData.size());
-  uint64_t CallStackBase = OS.tell();
-  for (const auto &[CSId, CallStack] : MemProfCallStackData) {
-    memprof::LinearCallStackId CallStackIndex =
-        (OS.tell() - CallStackBase) / sizeof(memprof::LinearCallStackId);
-    MemProfCallStackIndexes.insert({CSId, CallStackIndex});
-    const llvm::SmallVector<memprof::FrameId> CS = CallStack;
-    OS.write32(CS.size());
-    for (const auto F : CS) {
-      assert(MemProfFrameIndexes.contains(F));
-      OS.write32(MemProfFrameIndexes[F]);
-    }
-  }
+  memprof::CallStackRadixTreeBuilder Builder;
+  Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes,
+                FrameHistogram);
+  for (auto I : Builder.getRadixArray())
+    OS.write32(I);
+  MemProfCallStackIndexes = Builder.takeCallStackPos();
 
   // Release the memory of this vector as it is no longer needed.
   MemProfCallStackData.clear();
@@ -695,8 +712,8 @@ static Error writeMemProfV2(ProfOStream &OS,
 // uint64_t Schema entry 1
 // ....
 // uint64_t Schema entry N - 1
-// OnDiskChainedHashTable MemProfFrameData
-// OnDiskChainedHashTable MemProfCallStackData
+// Frames serialized one after another
+// Call stacks encoded as a radix tree
 // OnDiskChainedHashTable MemProfRecordData
 static Error writeMemProfV3(ProfOStream &OS,
                             memprof::IndexedMemProfData &MemProfData,
@@ -712,13 +729,17 @@ static Error writeMemProfV3(ProfOStream &OS,
     Schema = memprof::getFullSchema();
   writeMemProfSchema(OS, Schema);
 
+  llvm::DenseMap<memprof::FrameId, memprof::FrameStat> FrameHistogram =
+      memprof::computeFrameHistogram(MemProfData.CallStackData);
+  assert(MemProfData.FrameData.size() == FrameHistogram.size());
+
   llvm::DenseMap<memprof::FrameId, memprof::LinearFrameId> MemProfFrameIndexes =
-      writeMemProfFrameArray(OS, MemProfData.FrameData);
+      writeMemProfFrameArray(OS, MemProfData.FrameData, FrameHistogram);
 
   uint64_t CallStackPayloadOffset = OS.tell();
   llvm::DenseMap<memprof::CallStackId, memprof::LinearCallStackId>
       MemProfCallStackIndexes = writeMemProfCallStackArray(
-          OS, MemProfData.CallStackData, MemProfFrameIndexes);
+          OS, MemProfData.CallStackData, MemProfFrameIndexes, FrameHistogram);
 
   uint64_t RecordPayloadOffset = OS.tell();
   uint64_t RecordTableOffset =
@@ -902,7 +923,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
                   BinaryIds.end());
 
-  for (auto BI : BinaryIds) {
+  for (const auto &BI : BinaryIds) {
     // Increment by binary id length data type size.
     BinaryIdsSectionSize += sizeof(uint64_t);
     // Increment by binary id data length, aligned to 8 bytes.
@@ -911,7 +932,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   // Write binary ids section size.
   OS.write(BinaryIdsSectionSize);
 
-  for (auto BI : BinaryIds) {
+  for (const auto &BI : BinaryIds) {
     uint64_t BILen = BI.size();
     // Write binary id length.
     OS.write(BILen);
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 1d9860e0ea7e846e3056da4fe74e7556542fb538..4ca868722ec4e3c95b455c5109b05a991a3be57d 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -194,7 +194,8 @@ static void serializeV3(
 
 void IndexedMemProfRecord::serialize(
     const MemProfSchema &Schema, raw_ostream &OS, IndexedVersion Version,
-    llvm::DenseMap<CallStackId, LinearCallStackId> *MemProfCallStackIndexes) {
+    llvm::DenseMap<CallStackId, LinearCallStackId> *MemProfCallStackIndexes)
+    const {
   switch (Version) {
   case Version0:
   case Version1:
@@ -338,20 +339,19 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
 }
 
 MemProfRecord IndexedMemProfRecord::toMemProfRecord(
-    llvm::function_ref<llvm::SmallVector<Frame>(const CallStackId)> Callback)
-    const {
+    llvm::function_ref<std::vector<Frame>(const CallStackId)> Callback) const {
   MemProfRecord Record;
 
   Record.AllocSites.reserve(AllocSites.size());
-  for (const memprof::IndexedAllocationInfo &IndexedAI : AllocSites) {
-    memprof::AllocationInfo AI;
+  for (const IndexedAllocationInfo &IndexedAI : AllocSites) {
+    AllocationInfo AI;
     AI.Info = IndexedAI.Info;
     AI.CallStack = Callback(IndexedAI.CSId);
     Record.AllocSites.push_back(std::move(AI));
   }
 
   Record.CallSites.reserve(CallSiteIds.size());
-  for (memprof::CallStackId CSId : CallSiteIds)
+  for (CallStackId CSId : CallSiteIds)
     Record.CallSites.push_back(Callback(CSId));
 
   return Record;
@@ -410,6 +410,224 @@ CallStackId hashCallStack(ArrayRef<FrameId> CS) {
   return CSId;
 }
 
+// Encode a call stack into RadixArray.  Return the starting index within
+// RadixArray.  For each call stack we encode, we emit two or three components
+// into RadixArray.  If a given call stack doesn't have a common prefix relative
+// to the previous one, we emit:
+//
+// - the frames in the given call stack in the root-to-leaf order
+//
+// - the length of the given call stack
+//
+// If a given call stack has a non-empty common prefix relative to the previous
+// one, we emit:
+//
+// - the relative location of the common prefix, encoded as a negative number.
+//
+// - a portion of the given call stack that's beyond the common prefix
+//
+// - the length of the given call stack, including the length of the common
+//   prefix.
+//
+// The resulting RadixArray requires a somewhat unintuitive backward traversal
+// to reconstruct a call stack -- read the call stack length and scan backward
+// while collecting frames in the leaf to root order.  build, the caller of this
+// function, reverses RadixArray in place so that we can reconstruct a call
+// stack as if we were deserializing an array in a typical way -- the call stack
+// length followed by the frames in the leaf-to-root order except that we need
+// to handle pointers to parents along the way.
+//
+// To quickly determine the location of the common prefix within RadixArray,
+// Indexes caches the indexes of the previous call stack's frames within
+// RadixArray.
+LinearCallStackId CallStackRadixTreeBuilder::encodeCallStack(
+    const llvm::SmallVector<FrameId> *CallStack,
+    const llvm::SmallVector<FrameId> *Prev,
+    const llvm::DenseMap<FrameId, LinearFrameId> &MemProfFrameIndexes) {
+  // Compute the length of the common root prefix between Prev and CallStack.
+  uint32_t CommonLen = 0;
+  if (Prev) {
+    auto Pos = std::mismatch(Prev->rbegin(), Prev->rend(), CallStack->rbegin(),
+                             CallStack->rend());
+    CommonLen = std::distance(CallStack->rbegin(), Pos.second);
+  }
+
+  // Drop the portion beyond CommonLen.
+  assert(CommonLen <= Indexes.size());
+  Indexes.resize(CommonLen);
+
+  // Append a pointer to the parent.
+  if (CommonLen) {
+    uint32_t CurrentIndex = RadixArray.size();
+    uint32_t ParentIndex = Indexes.back();
+    // The offset to the parent must be negative because we are pointing to an
+    // element we've already added to RadixArray.
+    assert(ParentIndex < CurrentIndex);
+    RadixArray.push_back(ParentIndex - CurrentIndex);
+  }
+
+  // Copy the part of the call stack beyond the common prefix to RadixArray.
+  assert(CommonLen <= CallStack->size());
+  for (FrameId F : llvm::drop_begin(llvm::reverse(*CallStack), CommonLen)) {
+    // Remember the index of F in RadixArray.
+    Indexes.push_back(RadixArray.size());
+    RadixArray.push_back(MemProfFrameIndexes.find(F)->second);
+  }
+  assert(CallStack->size() == Indexes.size());
+
+  // End with the call stack length.
+  RadixArray.push_back(CallStack->size());
+
+  // Return the index within RadixArray where we can start reconstructing a
+  // given call stack from.
+  return RadixArray.size() - 1;
+}
+
+void CallStackRadixTreeBuilder::build(
+    llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
+        &&MemProfCallStackData,
+    const llvm::DenseMap<FrameId, LinearFrameId> &MemProfFrameIndexes,
+    llvm::DenseMap<FrameId, FrameStat> &FrameHistogram) {
+  // Take the vector portion of MemProfCallStackData.  The vector is exactly
+  // what we need to sort.  Also, we no longer need its lookup capability.
+  llvm::SmallVector<CSIdPair, 0> CallStacks = MemProfCallStackData.takeVector();
+
+  // Return early if we have no work to do.
+  if (CallStacks.empty()) {
+    RadixArray.clear();
+    CallStackPos.clear();
+    return;
+  }
+
+  // Sorting the list of call stacks in the dictionary order is sufficient to
+  // maximize the length of the common prefix between two adjacent call stacks
+  // and thus minimize the length of RadixArray.  However, we go one step
+  // further and try to reduce the number of times we follow pointers to parents
+  // during deserilization.  Consider a poorly encoded radix tree:
+  //
+  // CallStackId 1:  f1 -> f2 -> f3
+  //                  |
+  // CallStackId 2:   +--- f4 -> f5
+  //                        |
+  // CallStackId 3:         +--> f6
+  //
+  // Here, f2 and f4 appear once and twice, respectively, in the call stacks.
+  // Once we encode CallStackId 1 into RadixArray, every other call stack with
+  // common prefix f1 ends up pointing to CallStackId 1.  Since CallStackId 3
+  // share "f1 f4" with CallStackId 2, CallStackId 3 needs to follow pointers to
+  // parents twice.
+  //
+  // We try to alleviate the situation by sorting the list of call stacks by
+  // comparing the popularity of frames rather than the integer values of
+  // FrameIds.  In the example above, f4 is more popular than f2, so we sort the
+  // call stacks and encode them as:
+  //
+  // CallStackId 2:  f1 -- f4 -> f5
+  //                  |     |
+  // CallStackId 3:   |     +--> f6
+  //                  |
+  // CallStackId 1:   +--> f2 -> f3
+  //
+  // Notice that CallStackId 3 follows a pointer to a parent only once.
+  //
+  // All this is a quick-n-dirty trick to reduce the number of jumps.  The
+  // proper way would be to compute the weight of each radix tree node -- how
+  // many call stacks use a given radix tree node, and encode a radix tree from
+  // the heaviest node first.  We do not do so because that's a lot of work.
+  llvm::sort(CallStacks, [&](const CSIdPair &L, const CSIdPair &R) {
+    // Call stacks are stored from leaf to root.  Perform comparisons from the
+    // root.
+    return std::lexicographical_compare(
+        L.second.rbegin(), L.second.rend(), R.second.rbegin(), R.second.rend(),
+        [&](FrameId F1, FrameId F2) {
+          uint64_t H1 = FrameHistogram[F1].Count;
+          uint64_t H2 = FrameHistogram[F2].Count;
+          // Popular frames should come later because we encode call stacks from
+          // the last one in the list.
+          if (H1 != H2)
+            return H1 < H2;
+          // For sort stability.
+          return F1 < F2;
+        });
+  });
+
+  // Reserve some reasonable amount of storage.
+  RadixArray.clear();
+  RadixArray.reserve(CallStacks.size() * 8);
+
+  // Indexes will grow as long as the longest call stack.
+  Indexes.clear();
+  Indexes.reserve(512);
+
+  // CallStackPos will grow to exactly CallStacks.size() entries.
+  CallStackPos.clear();
+  CallStackPos.reserve(CallStacks.size());
+
+  // Compute the radix array.  We encode one call stack at a time, computing the
+  // longest prefix that's shared with the previous call stack we encode.  For
+  // each call stack we encode, we remember a mapping from CallStackId to its
+  // position within RadixArray.
+  //
+  // As an optimization, we encode from the last call stack in CallStacks to
+  // reduce the number of times we follow pointers to the parents.  Consider the
+  // list of call stacks that has been sorted in the dictionary order:
+  //
+  // Call Stack 1: F1
+  // Call Stack 2: F1 -> F2
+  // Call Stack 3: F1 -> F2 -> F3
+  //
+  // If we traversed CallStacks in the forward order, we would end up with a
+  // radix tree like:
+  //
+  // Call Stack 1:  F1
+  //                |
+  // Call Stack 2:  +---> F2
+  //                      |
+  // Call Stack 3:        +---> F3
+  //
+  // Notice that each call stack jumps to the previous one.  However, if we
+  // traverse CallStacks in the reverse order, then Call Stack 3 has the
+  // complete call stack encoded without any pointers.  Call Stack 1 and 2 point
+  // to appropriate prefixes of Call Stack 3.
+  const llvm::SmallVector<FrameId> *Prev = nullptr;
+  for (const auto &[CSId, CallStack] : llvm::reverse(CallStacks)) {
+    LinearCallStackId Pos =
+        encodeCallStack(&CallStack, Prev, MemProfFrameIndexes);
+    CallStackPos.insert({CSId, Pos});
+    Prev = &CallStack;
+  }
+
+  // "RadixArray.size() - 1" below is problematic if RadixArray is empty.
+  assert(!RadixArray.empty());
+
+  // Reverse the radix array in place.  We do so mostly for intuitive
+  // deserialization where we would read the length field and then the call
+  // stack frames proper just like any other array deserialization, except
+  // that we have occasional jumps to take advantage of prefixes.
+  for (size_t I = 0, J = RadixArray.size() - 1; I < J; ++I, --J)
+    std::swap(RadixArray[I], RadixArray[J]);
+
+  // "Reverse" the indexes stored in CallStackPos.
+  for (auto &[K, V] : CallStackPos)
+    V = RadixArray.size() - 1 - V;
+}
+
+llvm::DenseMap<FrameId, FrameStat>
+computeFrameHistogram(llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
+                          &MemProfCallStackData) {
+  llvm::DenseMap<FrameId, FrameStat> Histogram;
+
+  for (const auto &KV : MemProfCallStackData) {
+    const auto &CS = KV.second;
+    for (unsigned I = 0, E = CS.size(); I != E; ++I) {
+      auto &S = Histogram[CS[I]];
+      ++S.Count;
+      S.PositionSum += I;
+    }
+  }
+  return Histogram;
+}
+
 void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record) {
   for (const auto &AS : Record.AllocSites) {
     assert(AS.CSId == hashCallStack(AS.CallStack));
diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index fc3be716087ebeff3b710694bfc25d88347f896b..de58cb6331860bf196abb394504824aea4eaadbd 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -67,7 +67,7 @@ Error checkBuffer(const MemoryBuffer &Buffer) {
   uint64_t TotalSize = 0;
   const char *Next = Buffer.getBufferStart();
   while (Next < Buffer.getBufferEnd()) {
-    auto *H = reinterpret_cast<const Header *>(Next);
+    const auto *H = reinterpret_cast<const Header *>(Next);
     if (H->Version != MEMPROF_RAW_VERSION) {
       return make_error<InstrProfError>(instrprof_error::unsupported_version);
     }
@@ -597,7 +597,7 @@ RawMemProfReader::peekBuildIds(MemoryBuffer *DataBuffer) {
                   llvm::SmallSet<std::string, 10>>
       BuildIds;
   while (Next < DataBuffer->getBufferEnd()) {
-    auto *Header = reinterpret_cast<const memprof::Header *>(Next);
+    const auto *Header = reinterpret_cast<const memprof::Header *>(Next);
 
     const llvm::SmallVector<SegmentEntry> Entries =
         readSegmentEntries(Next + Header->SegmentOffset);
@@ -615,7 +615,7 @@ Error RawMemProfReader::readRawProfile(
   const char *Next = DataBuffer->getBufferStart();
 
   while (Next < DataBuffer->getBufferEnd()) {
-    auto *Header = reinterpret_cast<const memprof::Header *>(Next);
+    const auto *Header = reinterpret_cast<const memprof::Header *>(Next);
 
     // Read in the segment information, check whether its the same across all
     // profiles in this binary file.
@@ -690,7 +690,7 @@ Error RawMemProfReader::readNextRecord(
       return F;
     auto Iter = this->GuidToSymbolName.find(F.Function);
     assert(Iter != this->GuidToSymbolName.end());
-    F.SymbolName = Iter->getSecond();
+    F.SymbolName = std::make_unique<std::string>(Iter->getSecond());
     return F;
   };
   return MemProfReader::readNextRecord(GuidRecord, IdToFrameCallback);
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 625e523f13cec0e6f42ea2db11958c724e45fd6d..1630fefb4fcfbaf92492beed867aa613f9260d27 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -811,8 +811,7 @@ std::error_code SampleProfileWriterBinary::writeSummary() {
   encodeULEB128(Summary->getMaxFunctionCount(), OS);
   encodeULEB128(Summary->getNumCounts(), OS);
   encodeULEB128(Summary->getNumFunctions(), OS);
-  const std::vector<ProfileSummaryEntry> &Entries =
-      Summary->getDetailedSummary();
+  ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
   encodeULEB128(Entries.size(), OS);
   for (auto Entry : Entries) {
     encodeULEB128(Entry.Cutoff, OS);
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index d6012a8eea8a6ee39d682622c9b81e05b7036a58..e3ad1468874cde744362505df4e64ad26a618c36 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -18,11 +18,8 @@
 
 using namespace llvm;
 
-static KnownBits computeForAddCarry(
-    const KnownBits &LHS, const KnownBits &RHS,
-    bool CarryZero, bool CarryOne) {
-  assert(!(CarryZero && CarryOne) &&
-         "Carry can't be zero and one at the same time");
+static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS,
+                                    bool CarryZero, bool CarryOne) {
 
   APInt PossibleSumZero = LHS.getMaxValue() + RHS.getMaxValue() + !CarryZero;
   APInt PossibleSumOne = LHS.getMinValue() + RHS.getMinValue() + CarryOne;
@@ -37,9 +34,6 @@ static KnownBits computeForAddCarry(
   APInt CarryKnownUnion = std::move(CarryKnownZero) | CarryKnownOne;
   APInt Known = std::move(LHSKnownUnion) & RHSKnownUnion & CarryKnownUnion;
 
-  assert((PossibleSumZero & Known) == (PossibleSumOne & Known) &&
-         "known bits of sum differ");
-
   // Compute known bits of the result.
   KnownBits KnownOut;
   KnownOut.Zero = ~std::move(PossibleSumZero) & Known;
@@ -608,14 +602,12 @@ KnownBits KnownBits::abs(bool IntMinIsPoison) const {
     }
   }
 
-  assert(!KnownAbs.hasConflict() && "Bad Output");
   return KnownAbs;
 }
 
 static KnownBits computeForSatAddSub(bool Add, bool Signed,
                                      const KnownBits &LHS,
                                      const KnownBits &RHS) {
-  assert(!LHS.hasConflict() && !RHS.hasConflict() && "Bad inputs");
   // We don't see NSW even for sadd/ssub as we want to check if the result has
   // signed overflow.
   KnownBits Res =
@@ -715,7 +707,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed,
     // We know whether or not we overflowed.
     if (!(*Overflow)) {
       // No overflow.
-      assert(!Res.hasConflict() && "Bad Output");
       return Res;
     }
 
@@ -737,7 +728,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed,
 
     Res.One = C;
     Res.Zero = ~C;
-    assert(!Res.hasConflict() && "Bad Output");
     return Res;
   }
 
@@ -757,7 +747,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed,
     Res.One.clearAllBits();
   }
 
-  assert(!Res.hasConflict() && "Bad Output");
   return Res;
 }
 
@@ -808,8 +797,7 @@ KnownBits KnownBits::avgCeilU(const KnownBits &LHS, const KnownBits &RHS) {
 KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
                          bool NoUndefSelfMultiply) {
   unsigned BitWidth = LHS.getBitWidth();
-  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
-         !RHS.hasConflict() && "Operand mismatch");
+  assert(BitWidth == RHS.getBitWidth() && "Operand mismatch");
   assert((!NoUndefSelfMultiply || LHS == RHS) &&
          "Self multiplication knownbits mismatch");
 
@@ -905,8 +893,7 @@ KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
 
 KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) {
   unsigned BitWidth = LHS.getBitWidth();
-  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
-         !RHS.hasConflict() && "Operand mismatch");
+  assert(BitWidth == RHS.getBitWidth() && "Operand mismatch");
   KnownBits WideLHS = LHS.sext(2 * BitWidth);
   KnownBits WideRHS = RHS.sext(2 * BitWidth);
   return mul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
@@ -914,8 +901,7 @@ KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) {
 
 KnownBits KnownBits::mulhu(const KnownBits &LHS, const KnownBits &RHS) {
   unsigned BitWidth = LHS.getBitWidth();
-  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
-         !RHS.hasConflict() && "Operand mismatch");
+  assert(BitWidth == RHS.getBitWidth() && "Operand mismatch");
   KnownBits WideLHS = LHS.zext(2 * BitWidth);
   KnownBits WideRHS = RHS.zext(2 * BitWidth);
   return mul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
@@ -964,7 +950,6 @@ KnownBits KnownBits::sdiv(const KnownBits &LHS, const KnownBits &RHS,
     return udiv(LHS, RHS, Exact);
 
   unsigned BitWidth = LHS.getBitWidth();
-  assert(!LHS.hasConflict() && !RHS.hasConflict() && "Bad inputs");
   KnownBits Known(BitWidth);
 
   if (LHS.isZero() || RHS.isZero()) {
@@ -1011,15 +996,12 @@ KnownBits KnownBits::sdiv(const KnownBits &LHS, const KnownBits &RHS,
   }
 
   Known = divComputeLowBit(Known, LHS, RHS, Exact);
-
-  assert(!Known.hasConflict() && "Bad Output");
   return Known;
 }
 
 KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS,
                           bool Exact) {
   unsigned BitWidth = LHS.getBitWidth();
-  assert(!LHS.hasConflict() && !RHS.hasConflict());
   KnownBits Known(BitWidth);
 
   if (LHS.isZero() || RHS.isZero()) {
@@ -1041,7 +1023,6 @@ KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS,
   Known.Zero.setHighBits(LeadZ);
   Known = divComputeLowBit(Known, LHS, RHS, Exact);
 
-  assert(!Known.hasConflict() && "Bad Output");
   return Known;
 }
 
@@ -1059,8 +1040,6 @@ KnownBits KnownBits::remGetLowBits(const KnownBits &LHS, const KnownBits &RHS) {
 }
 
 KnownBits KnownBits::urem(const KnownBits &LHS, const KnownBits &RHS) {
-  assert(!LHS.hasConflict() && !RHS.hasConflict());
-
   KnownBits Known = remGetLowBits(LHS, RHS);
   if (RHS.isConstant() && RHS.getConstant().isPowerOf2()) {
     // NB: Low bits set in `remGetLowBits`.
@@ -1078,8 +1057,6 @@ KnownBits KnownBits::urem(const KnownBits &LHS, const KnownBits &RHS) {
 }
 
 KnownBits KnownBits::srem(const KnownBits &LHS, const KnownBits &RHS) {
-  assert(!LHS.hasConflict() && !RHS.hasConflict());
-
   KnownBits Known = remGetLowBits(LHS, RHS);
   if (RHS.isConstant() && RHS.getConstant().isPowerOf2()) {
     // NB: Low bits are set in `remGetLowBits`.
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index fcefdef992be5c25260407ddbdc1aff9d968fb2a..7360901f2962c9d5c1bdaf1b774064d2f3e938e9 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -867,21 +867,16 @@ bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
   // Any intermediate directories we create should be accessible by
   // the owner, even if Perms says otherwise for the final path.
   const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all;
+
+  StringRef Name = *I;
   while (true) {
-    StringRef Name = *I;
-    detail::InMemoryNode *Node = Dir->getChild(Name);
+    Name = *I;
     ++I;
+    if (I == E)
+      break;
+    detail::InMemoryNode *Node = Dir->getChild(Name);
     if (!Node) {
-      if (I == E) {
-        // End of the path.
-        Dir->addChild(
-            Name, MakeNode({Dir->getUniqueID(), Path, Name, ModificationTime,
-                            std::move(Buffer), ResolvedUser, ResolvedGroup,
-                            ResolvedType, ResolvedPerms}));
-        return true;
-      }
-
-      // Create a new directory. Use the path up to here.
+      // This isn't the last element, so we create a new directory.
       Status Stat(
           StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
           getDirectoryID(Dir->getUniqueID(), Name),
@@ -891,27 +886,33 @@ bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
           Name, std::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
       continue;
     }
+    // Creating file under another file.
+    if (!isa<detail::InMemoryDirectory>(Node))
+      return false;
+    Dir = cast<detail::InMemoryDirectory>(Node);
+  }
+  detail::InMemoryNode *Node = Dir->getChild(Name);
+  if (!Node) {
+    Dir->addChild(Name,
+                  MakeNode({Dir->getUniqueID(), Path, Name, ModificationTime,
+                            std::move(Buffer), ResolvedUser, ResolvedGroup,
+                            ResolvedType, ResolvedPerms}));
+    return true;
+  }
+  if (isa<detail::InMemoryDirectory>(Node))
+    return ResolvedType == sys::fs::file_type::directory_file;
 
-    if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node)) {
-      Dir = NewDir;
-    } else {
-      assert((isa<detail::InMemoryFile>(Node) ||
-              isa<detail::InMemoryHardLink>(Node)) &&
-             "Must be either file, hardlink or directory!");
-
-      // Trying to insert a directory in place of a file.
-      if (I != E)
-        return false;
+  assert((isa<detail::InMemoryFile>(Node) ||
+          isa<detail::InMemoryHardLink>(Node)) &&
+         "Must be either file, hardlink or directory!");
 
-      // Return false only if the new file is different from the existing one.
-      if (auto Link = dyn_cast<detail::InMemoryHardLink>(Node)) {
-        return Link->getResolvedFile().getBuffer()->getBuffer() ==
-               Buffer->getBuffer();
-      }
-      return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
-             Buffer->getBuffer();
-    }
+  // Return false only if the new file is different from the existing one.
+  if (auto *Link = dyn_cast<detail::InMemoryHardLink>(Node)) {
+    return Link->getResolvedFile().getBuffer()->getBuffer() ==
+           Buffer->getBuffer();
   }
+  return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
+         Buffer->getBuffer();
 }
 
 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index 0f0a22ec82936106674cf05cd9de76456670566c..6f2aeb83a451af918a9ba372ed8dfb82ee5349d7 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -90,7 +90,6 @@ void initializeAArch64DeadRegisterDefinitionsPass(PassRegistry&);
 void initializeAArch64ExpandPseudoPass(PassRegistry &);
 void initializeAArch64GlobalsTaggingPass(PassRegistry &);
 void initializeAArch64LoadStoreOptPass(PassRegistry&);
-void initializeAArch64LoopIdiomTransformLegacyPassPass(PassRegistry &);
 void initializeAArch64LowerHomogeneousPrologEpilogPass(PassRegistry &);
 void initializeAArch64MIPeepholeOptPass(PassRegistry &);
 void initializeAArch64O0PreLegalizerCombinerPass(PassRegistry &);
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 4b2ce0d73949cb38623a785bf89fac59c7bf8b4e..5708b6173750a2e8c515ceee1fb80fb9eefeb4e5 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -85,6 +85,10 @@ def SMEUnsupported : AArch64Unsupported {
                       SME2Unsupported.F);
 }
 
+def MTEUnsupported : AArch64Unsupported {
+  let F = [HasMTE];
+}
+
 let F = [HasPAuth, HasPAuthLR] in
 def PAUnsupported : AArch64Unsupported;
 
@@ -109,6 +113,7 @@ include "AArch64SchedNeoverseN1.td"
 include "AArch64SchedNeoverseN2.td"
 include "AArch64SchedNeoverseV1.td"
 include "AArch64SchedNeoverseV2.td"
+include "AArch64SchedOryon.td"
 
 include "AArch64Processors.td"
 
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 32646c6ee689132fe336ba1175e76a0d03f6fd12..941990c53c4a7f2b87b5d47462bda0ba0c1f0b78 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -589,6 +589,14 @@ def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0
                                                  (sequence "X%u",19, 28),
                                                  LR, FP)>;
 
+// SME ABI support routines such as __arm_get_current_vg preserve most registers.
+def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1
+                          : CalleeSavedRegs<(add (sequence "Z%u", 0, 31),
+                                                 (sequence "P%u", 0, 15),
+                                                 (sequence "X%u", 1, 15),
+                                                 (sequence "X%u",19, 28),
+                                                 LR, FP)>;
+
 // SME ABI support routines __arm_sme_state preserves most registers.
 def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2
                           : CalleeSavedRegs<(add (sequence "Z%u", 0, 31),
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 62cf6a2c47accf75c471d7d7fb36dd8e17b1a845..56fd15f23363d1c0ddbbeb5c1735a852710883b6 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -3534,6 +3534,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
   }
   case Intrinsic::sin:
   case Intrinsic::cos:
+  case Intrinsic::tan:
   case Intrinsic::pow: {
     MVT RetVT;
     if (!isTypeLegal(II->getType(), RetVT))
@@ -3542,11 +3543,11 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     if (RetVT != MVT::f32 && RetVT != MVT::f64)
       return false;
 
-    static const RTLIB::Libcall LibCallTable[3][2] = {
-      { RTLIB::SIN_F32, RTLIB::SIN_F64 },
-      { RTLIB::COS_F32, RTLIB::COS_F64 },
-      { RTLIB::POW_F32, RTLIB::POW_F64 }
-    };
+    static const RTLIB::Libcall LibCallTable[4][2] = {
+        {RTLIB::SIN_F32, RTLIB::SIN_F64},
+        {RTLIB::COS_F32, RTLIB::COS_F64},
+        {RTLIB::TAN_F32, RTLIB::TAN_F64},
+        {RTLIB::POW_F32, RTLIB::POW_F64}};
     RTLIB::Libcall LC;
     bool Is64Bit = RetVT == MVT::f64;
     switch (II->getIntrinsicID()) {
@@ -3558,9 +3559,12 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     case Intrinsic::cos:
       LC = LibCallTable[1][Is64Bit];
       break;
-    case Intrinsic::pow:
+    case Intrinsic::tan:
       LC = LibCallTable[2][Is64Bit];
       break;
+    case Intrinsic::pow:
+      LC = LibCallTable[3][Is64Bit];
+      break;
     }
 
     ArgListTy Args;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 360a841bdade485bbf174bbebe64608d88f8bf7e..48bf648b0052278c254e96a69baabff6333f4602 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -543,6 +543,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
   setOperationAction(ISD::FSQRT, MVT::f128, Expand);
   setOperationAction(ISD::FSUB, MVT::f128, LibCall);
+  setOperationAction(ISD::FTAN, MVT::f128, Expand);
   setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
   setOperationAction(ISD::SETCC, MVT::f128, Custom);
   setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
@@ -727,14 +728,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
   }
 
-  for (auto Op : {ISD::FREM,        ISD::FPOW,         ISD::FPOWI,
-                  ISD::FCOS,        ISD::FSIN,         ISD::FSINCOS,
-                  ISD::FEXP,        ISD::FEXP2,        ISD::FEXP10,
-                  ISD::FLOG,        ISD::FLOG2,        ISD::FLOG10,
-                  ISD::STRICT_FREM,
-                  ISD::STRICT_FPOW, ISD::STRICT_FPOWI, ISD::STRICT_FCOS,
-                  ISD::STRICT_FSIN, ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,
-                  ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) {
+  for (auto Op : {ISD::FREM,         ISD::FPOW,         ISD::FPOWI,
+                  ISD::FCOS,         ISD::FSIN,         ISD::FSINCOS,
+                  ISD::FTAN,         ISD::FEXP,         ISD::FEXP2,
+                  ISD::FEXP10,       ISD::FLOG,         ISD::FLOG2,
+                  ISD::FLOG10,       ISD::STRICT_FREM,  ISD::STRICT_FPOW,
+                  ISD::STRICT_FPOWI, ISD::STRICT_FCOS,  ISD::STRICT_FSIN,
+                  ISD::STRICT_FEXP,  ISD::STRICT_FEXP2, ISD::STRICT_FLOG,
+                  ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) {
     setOperationAction(Op, MVT::f16, Promote);
     setOperationAction(Op, MVT::v4f16, Expand);
     setOperationAction(Op, MVT::v8f16, Expand);
@@ -1171,26 +1172,27 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   if (Subtarget->isNeonAvailable()) {
     // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
     // silliness like this:
+    // clang-format off
     for (auto Op :
-         {ISD::SELECT,         ISD::SELECT_CC,
-          ISD::BR_CC,          ISD::FADD,           ISD::FSUB,
-          ISD::FMUL,           ISD::FDIV,           ISD::FMA,
-          ISD::FNEG,           ISD::FABS,           ISD::FCEIL,
-          ISD::FSQRT,          ISD::FFLOOR,         ISD::FNEARBYINT,
-          ISD::FSIN,           ISD::FCOS,           ISD::FPOW,
-          ISD::FLOG,           ISD::FLOG2,          ISD::FLOG10,
-          ISD::FEXP,           ISD::FEXP2,          ISD::FEXP10,
-          ISD::FRINT,          ISD::FROUND,         ISD::FROUNDEVEN,
-          ISD::FTRUNC,         ISD::FMINNUM,        ISD::FMAXNUM,
-          ISD::FMINIMUM,       ISD::FMAXIMUM,       ISD::STRICT_FADD,
-          ISD::STRICT_FSUB,    ISD::STRICT_FMUL,    ISD::STRICT_FDIV,
-          ISD::STRICT_FMA,     ISD::STRICT_FCEIL,   ISD::STRICT_FFLOOR,
-          ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,   ISD::STRICT_FNEARBYINT,
-          ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,  ISD::STRICT_FROUNDEVEN,
-          ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM,
-          ISD::STRICT_FMAXIMUM})
+         {ISD::SELECT,            ISD::SELECT_CC,
+          ISD::BR_CC,             ISD::FADD,           ISD::FSUB,
+          ISD::FMUL,              ISD::FDIV,           ISD::FMA,
+          ISD::FNEG,              ISD::FABS,           ISD::FCEIL,
+          ISD::FSQRT,             ISD::FFLOOR,         ISD::FNEARBYINT,
+          ISD::FSIN,              ISD::FCOS,           ISD::FTAN,
+          ISD::FPOW,              ISD::FLOG,           ISD::FLOG2,          
+          ISD::FLOG10,            ISD::FEXP,           ISD::FEXP2,
+          ISD::FEXP10,            ISD::FRINT,          ISD::FROUND,
+          ISD::FROUNDEVEN,        ISD::FTRUNC,         ISD::FMINNUM,
+          ISD::FMAXNUM,           ISD::FMINIMUM,       ISD::FMAXIMUM,
+          ISD::STRICT_FADD,       ISD::STRICT_FSUB,    ISD::STRICT_FMUL,
+          ISD::STRICT_FDIV,       ISD::STRICT_FMA,     ISD::STRICT_FCEIL,
+          ISD::STRICT_FFLOOR,     ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,
+          ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,  
+          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
+          ISD::STRICT_FMINIMUM,   ISD::STRICT_FMAXIMUM})
       setOperationAction(Op, MVT::v1f64, Expand);
-
+    // clang-format on
     for (auto Op :
          {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP,
           ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL,
@@ -1622,6 +1624,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FCOS, VT, Expand);
       setOperationAction(ISD::FSIN, VT, Expand);
       setOperationAction(ISD::FSINCOS, VT, Expand);
+      setOperationAction(ISD::FTAN, VT, Expand);
       setOperationAction(ISD::FEXP, VT, Expand);
       setOperationAction(ISD::FEXP2, VT, Expand);
       setOperationAction(ISD::FEXP10, VT, Expand);
@@ -1803,6 +1806,7 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
   if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
     setOperationAction(ISD::FSIN, VT, Expand);
     setOperationAction(ISD::FCOS, VT, Expand);
+    setOperationAction(ISD::FTAN, VT, Expand);
     setOperationAction(ISD::FPOW, VT, Expand);
     setOperationAction(ISD::FLOG, VT, Expand);
     setOperationAction(ISD::FLOG2, VT, Expand);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 081d64921ee1ce4d2521e083f12006d9ae7f5603..91e5bc3caa102cf84ed57356709e032deb2df570 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -9405,6 +9405,31 @@ def : Pat<(AArch64faddp (v4f16 (extract_subvector (v8f16 FPR128:$Rn), (i64 0))),
                         (v4f16 (extract_subvector (v8f16 FPR128:$Rn), (i64 4)))),
           (v4f16 (EXTRACT_SUBREG (FADDPv8f16 $Rn, $Rn), dsub))>;
 
+// add(uzp1(X, Y), uzp2(X, Y)) -> addp(X, Y)
+def : Pat<(v2i64 (add (AArch64zip1 (v2i64 FPR128:$Rn), (v2i64 FPR128:$Rm)),
+                      (AArch64zip2 (v2i64 FPR128:$Rn), (v2i64 FPR128:$Rm)))),
+          (v2i64 (ADDPv2i64 $Rn, $Rm))>;
+def : Pat<(v4i32 (add (AArch64uzp1 (v4i32 FPR128:$Rn), (v4i32 FPR128:$Rm)),
+                      (AArch64uzp2 (v4i32 FPR128:$Rn), (v4i32 FPR128:$Rm)))),
+          (v4i32 (ADDPv4i32 $Rn, $Rm))>;
+def : Pat<(v8i16 (add (AArch64uzp1 (v8i16 FPR128:$Rn), (v8i16 FPR128:$Rm)),
+                      (AArch64uzp2 (v8i16 FPR128:$Rn), (v8i16 FPR128:$Rm)))),
+          (v8i16 (ADDPv8i16 $Rn, $Rm))>;
+def : Pat<(v16i8 (add (AArch64uzp1 (v16i8 FPR128:$Rn), (v16i8 FPR128:$Rm)),
+                      (AArch64uzp2 (v16i8 FPR128:$Rn), (v16i8 FPR128:$Rm)))),
+          (v16i8 (ADDPv16i8 $Rn, $Rm))>;
+
+def : Pat<(v2f64 (fadd (AArch64zip1 (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm)),
+                       (AArch64zip2 (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm)))),
+          (v2f64 (FADDPv2f64 $Rn, $Rm))>;
+def : Pat<(v4f32 (fadd (AArch64uzp1 (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm)),
+                       (AArch64uzp2 (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm)))),
+          (v4f32 (FADDPv4f32 $Rn, $Rm))>;
+let Predicates = [HasFullFP16] in
+def : Pat<(v8f16 (fadd (AArch64uzp1 (v8f16 FPR128:$Rn), (v8f16 FPR128:$Rm)),
+                       (AArch64uzp2 (v8f16 FPR128:$Rn), (v8f16 FPR128:$Rm)))),
+          (v8f16 (FADDPv8f16 $Rn, $Rm))>;
+
 // Scalar 64-bit shifts in FPR64 registers.
 def : Pat<(i64 (int_aarch64_neon_sshl (i64 FPR64:$Rn), (i64 FPR64:$Rm))),
           (SSHLv1i64 FPR64:$Rn, FPR64:$Rm)>;
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 8d16709114dfc4e168e0cc3ace8722530cf256a5..a759efcd9441ec3098dd91ca50a44dcbd567208f 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -617,6 +617,27 @@ def TuneAmpere1B : SubtargetFeature<"ampere1b", "ARMProcFamily", "Ampere1B",
                                     FeatureLdpAlignedOnly,
                                     FeatureStpAlignedOnly]>;
 
+def TuneOryon  : SubtargetFeature<"oryon-1", "ARMProcFamily",
+                                    "Oryon",
+                                    "Nuvia Inc Oryon processors", [
+                                    FeatureCrypto,
+                                    FeatureFPARMv8,
+                                    FeatureNEON,
+                                    FeatureFuseAES,
+                                    FeatureFuseAdrpAdd,
+                                    FeatureEnableSelectOptimize,
+                                    FeatureFuseCryptoEOR,
+                                    FeatureFuseAddress,
+                                    FeatureSM4,
+                                    FeatureSHA2,
+                                    FeatureSHA3,
+                                    FeatureAES,
+                                    FeatureFullFP16,
+                                    FeatureFP16FML,
+                                    FeaturePerfMon,
+                                    FeatureSPE,
+                                    FeaturePostRAScheduler,
+                                    HasV8_6aOps]>;
 
 def ProcessorFeatures {
   list<SubtargetFeature> A53  = [HasV8_0aOps, FeatureCRC, FeatureCrypto,
@@ -806,6 +827,11 @@ def ProcessorFeatures {
                                      FeatureSHA3, FeatureAES, FeatureCSSC,
                                      FeatureWFxT, FeatureFullFP16];
 
+  list<SubtargetFeature> Oryon = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
+                                     FeatureCrypto, FeatureRandGen,
+                                     FeaturePAuth, FeatureSM4, FeatureSHA2,
+                                     FeatureSHA3, FeatureAES];
+
   // ETE and TRBE are future architecture extensions. We temporarily enable them
   // by default for users targeting generic AArch64. The extensions do not
   // affect code generated by the compiler and can be used only by explicitly
@@ -988,3 +1014,7 @@ def : ProcessorModel<"ampere1a", Ampere1Model, ProcessorFeatures.Ampere1A,
 
 def : ProcessorModel<"ampere1b", Ampere1BModel, ProcessorFeatures.Ampere1B,
                      [TuneAmpere1B]>;
+
+// Qualcomm Oryon
+def : ProcessorModel<"oryon-1", OryonModel, ProcessorFeatures.Oryon,
+                       [TuneOryon]>;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index e97d7e3b6ed81ee6bdb0033587e64ca373cb7f63..cc50b59dd8d7e1267f34c81f0d4d449d4414e6f2 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -107,13 +107,22 @@ AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   if (MF->getFunction().getCallingConv() ==
           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
     report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
-        "only supported to improve calls to SME ACLE save/restore/disable-za "
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only "
+        "supported to improve calls to SME ACLE save/restore/disable-za "
         "functions, and is not intended to be used beyond that scope.");
+  if (MF->getFunction().getCallingConv() ==
+      CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
+    report_fatal_error(
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is "
+        "only supported to improve calls to SME ACLE __arm_get_current_vg "
+        "function, and is not intended to be used beyond that scope.");
   if (MF->getFunction().getCallingConv() ==
           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
     report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
         "only supported to improve calls to SME ACLE __arm_sme_state "
         "and is not intended to be used beyond that scope.");
   if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
@@ -153,13 +162,22 @@ AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const {
   if (MF->getFunction().getCallingConv() ==
           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
     report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
         "only supported to improve calls to SME ACLE save/restore/disable-za "
         "functions, and is not intended to be used beyond that scope.");
+  if (MF->getFunction().getCallingConv() ==
+      CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
+    report_fatal_error(
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is "
+        "only supported to improve calls to SME ACLE __arm_get_current_vg "
+        "function, and is not intended to be used beyond that scope.");
   if (MF->getFunction().getCallingConv() ==
           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
     report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
+        "Calling convention "
+        "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
         "only supported to improve calls to SME ACLE __arm_sme_state "
         "and is not intended to be used beyond that scope.");
   if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS)
@@ -236,6 +254,8 @@ AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
         "Calling convention SVE_VectorCall is unsupported on Darwin.");
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
+  if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
+    return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
   if (CC == CallingConv::CFGuard_Check)
@@ -282,6 +302,8 @@ AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
                : CSR_AArch64_SVE_AAPCS_RegMask;
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
+  if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
+    return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
   if (CC == CallingConv::CFGuard_Check)
@@ -643,6 +665,7 @@ bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
   case CallingConv::AArch64_VectorCall:
   case CallingConv::AArch64_SVE_VectorCall:
   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
+  case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
     if (STI.isTargetWindows())
       return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
diff --git a/llvm/lib/Target/AArch64/AArch64SchedOryon.td b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
new file mode 100644
index 0000000000000000000000000000000000000000..09d1af248f0ec34128370e6984dd69d41f9e3bb2
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
@@ -0,0 +1,1659 @@
+//=- AArch64SchedOryon.td - Qualcomm Oryon CPU 001 ---*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the scheduling model for Qualcomm Oryon
+// family of processors.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Pipeline Description.
+
+def OryonModel : SchedMachineModel {
+  let IssueWidth            =  14;
+  let MicroOpBufferSize     = 376;
+  let LoadLatency           =   4;
+  let MispredictPenalty     =  13; // 13 cycles for mispredicted branch.
+  let LoopMicroOpBufferSize =   0; // Do not have a LoopMicroOpBuffer
+  let PostRAScheduler       =   1; // Using PostRA sched.
+  let CompleteModel         =   1;
+
+  list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
+                                                    SMEUnsupported.F,
+                                                    MTEUnsupported.F,
+                                                    PAUnsupported.F,
+                                                    [HasPAuth, HasCSSC]);
+}
+
+let SchedModel = OryonModel in {
+
+// Issue ports.
+// IXU has 6 ports p0 ~ p5
+// LSU has 4 ports p6 ~ p9(ls0 ~ ls3), p10/p11(std0, std1) has to work with ls0~ls3
+// VXU has 4 ports p12 ~ p15
+
+// cross IXU/LSU/VXU resource group for FMOV P41 of VXU
+// I2V
+def ORYONI4FP0 : ProcResource<1>;
+def ORYONI5FP1 : ProcResource<1>;
+// V2I
+def ORYONFP0I4 : ProcResource<1>;
+def ORYONFP1I5 : ProcResource<1>;
+
+// store 1 for normal store instructions
+def ORYONST0 : ProcResource<1>;
+// store 2 for normal store instructions
+def ORYONST1 : ProcResource<1>;
+
+// Port 0: ALU/Indirect/Direct Branch.
+def ORYONP0 : ProcResource<1>;
+
+// Port 1: ALU/Direct Branch.
+def ORYONP1 : ProcResource<1>;
+
+// Port 2: ALU.
+def ORYONP2 : ProcResource<1>;
+
+// Port 3: ALU.
+def ORYONP3 : ProcResource<1>;
+
+// Port 4: ALU.
+def ORYONP4 : ProcResource<1> {
+    let Super = ORYONI4FP0;
+    let Super = ORYONFP0I4; }
+
+// Port 5: ALU.
+def ORYONP5 : ProcResource<1> {
+    let Super = ORYONI5FP1;
+    let Super = ORYONFP1I5; }
+
+// Port 6: Load/Store. LS0
+def ORYONP6 : ProcResource<1> {
+    let Super = ORYONST0; }
+
+// Port 7: Load/store. LS1
+def ORYONP7 : ProcResource<1> {
+    let Super = ORYONST0; }
+
+// Port 8: Load/Store. LS2
+def ORYONP8 : ProcResource<1> {
+    let Super = ORYONST1; }
+
+// Port 9: Load/store. LS3
+def ORYONP9 : ProcResource<1> {
+    let Super = ORYONST1; }
+
+// Port 10: Load/Store. STD0
+def ORYONP10SD0 : ProcResource<1> {
+    let Super = ORYONST0; }
+
+// Port 11: Load/store. STD1
+def ORYONP11SD1 : ProcResource<1> {
+    let Super = ORYONST1; }
+
+// Port 12: FP/Neon/SIMD/Crypto.
+def ORYONP12FP0 : ProcResource<1> {
+    let Super = ORYONI4FP0;
+    let Super = ORYONFP0I4; }
+
+// Port 13: FP/Neon/SIMD/Crypto.
+def ORYONP13FP1 : ProcResource<1> {
+    let Super = ORYONI5FP1;
+    let Super = ORYONFP1I5; }
+
+// Port 14: FP/Neon/SIMD/Crypto.
+def ORYONP14FP2 : ProcResource<1>;
+
+// Port 15: FP/Neon/SIMD/Crypto.
+def ORYONP15FP3 : ProcResource<1>;
+
+// Define groups for the functional units on each issue port.  Each group
+// created will be used by a WriteRes.
+
+// Integer add/shift/logical/misc. instructions on port I0/I1/I2/I3/I4/I5.
+def ORYONI012345 : ProcResGroup<[ORYONP0, ORYONP1, ORYONP2,
+                                  ORYONP3, ORYONP4, ORYONP5]> {
+  let BufferSize = 120;
+}
+
+// Direct Conditional Branch instructions on ports I0/I1.
+def ORYONI01 : ProcResGroup<[ORYONP0, ORYONP1]> {
+  let BufferSize = 40;
+}
+
+// Indirect/crypto Conditional Branch instructions on ports I0.
+def ORYONI0 : ProcResGroup<[ORYONP0]> {
+  let BufferSize = 20;
+}
+
+// Crypto/CRC/PAU instructions on ports I2.
+def ORYONI2 : ProcResGroup<[ORYONP2]> {
+  let BufferSize = 20;
+}
+
+// Multiply/Multiply-ADD instructions on ports I4/I5.
+def ORYONI45 : ProcResGroup<[ORYONP4, ORYONP5]> {
+  let BufferSize = 40;
+}
+
+// Divide instructions on ports I5.
+def ORYONI5 : ProcResGroup<[ORYONP5]> {
+  let BufferSize = 20;
+}
+
+// Comparison instructions on ports I0/I1/I2/I3.
+def ORYONI0123 : ProcResGroup<[ORYONP0, ORYONP1,
+                                ORYONP2, ORYONP3]> {
+  let BufferSize = 80;
+}
+
+// Load instructions on ports P6/P7/P8/P9.
+def ORYONLD : ProcResGroup<[ORYONP6, ORYONP7, ORYONP8, ORYONP9]> {
+  let BufferSize = 64;
+}
+
+// Store instructions on combo of STA/STD pipes
+def ORYONST : ProcResGroup<[ORYONST0, ORYONST1]> {
+    let BufferSize = 64;
+}
+
+// Arithmetic and CRYP-AED ASIMD/FP instructions on ports FP0/FP1/FP2/FP3.
+def ORYONFP0123 : ProcResGroup<[ORYONP12FP0, ORYONP13FP1,
+                                   ORYONP14FP2, ORYONP15FP3]> {
+  let BufferSize = 192;
+}
+
+// FP Comparison and F/I move instructions on ports FP0/FP1.
+def ORYONFP01 : ProcResGroup<[ORYONP12FP0, ORYONP13FP1]> {
+  let BufferSize = 96;
+}
+
+// FDIV instructions on ports FP3.
+def ORYONFP3 : ProcResGroup<[ORYONP15FP3]> {
+  let BufferSize = 48;
+}
+
+// CRYP-SHA instructions on ports FP1.
+def ORYONFP1 : ProcResGroup<[ORYONP14FP2]> {
+  let BufferSize = 48;
+}
+
+def ORYONFP2 : ProcResGroup<[ORYONP14FP2]> {
+  let BufferSize = 48;
+}
+
+// Reciprocal, Squre root on FP0.
+def ORYONFP0 : ProcResGroup<[ORYONP12FP0]> {
+  let BufferSize = 48;
+}
+
+// cross IXU/LSU/VXU resource group for FMOV P41 of VXU
+// I2V
+def ORYONI2V : ProcResGroup<[ORYONI4FP0, ORYONI5FP1]> {
+    let BufferSize = 40;
+}
+
+// V2I
+def ORYONV2I : ProcResGroup<[ORYONFP0I4, ORYONFP1I5]> {
+    let BufferSize = 96;
+}
+
+// Define commonly used write types for InstRW specializations.
+// All definitions follow the format: ORYONWrite_<NumCycles>Cyc_<Resources>.
+
+// Because of the complexity of Oryon CPU, we skip the following
+// generic definitions and define each instruction specifically
+
+// These WriteRes entries are not used in the Falkor sched model.
+def : WriteRes<WriteImm, []>     { let Unsupported = 1; }
+def : WriteRes<WriteI, []>       { let Unsupported = 1; }
+def : WriteRes<WriteISReg, []>   { let Unsupported = 1; }
+def : WriteRes<WriteIEReg, []>   { let Unsupported = 1; }
+def : WriteRes<WriteExtr, []>    { let Unsupported = 1; }
+def : WriteRes<WriteIS, []>      { let Unsupported = 1; }
+def : WriteRes<WriteID32, []>    { let Unsupported = 1; }
+def : WriteRes<WriteID64, []>    { let Unsupported = 1; }
+def : WriteRes<WriteIM32, []>    { let Unsupported = 1; }
+def : WriteRes<WriteIM64, []>    { let Unsupported = 1; }
+def : WriteRes<WriteBr, []>      { let Unsupported = 1; }
+def : WriteRes<WriteBrReg, []>   { let Unsupported = 1; }
+def : WriteRes<WriteLD, []>      { let Unsupported = 1; }
+def : WriteRes<WriteST, []>      { let Unsupported = 1; }
+def : WriteRes<WriteSTP, []>     { let Unsupported = 1; }
+def : WriteRes<WriteAdr, []>     { let Unsupported = 1; }
+def : WriteRes<WriteLDIdx, []>   { let Unsupported = 1; }
+def : WriteRes<WriteSTIdx, []>   { let Unsupported = 1; }
+def : WriteRes<WriteF, []>       { let Unsupported = 1; }
+def : WriteRes<WriteFCmp, []>    { let Unsupported = 1; }
+def : WriteRes<WriteFCvt, []>    { let Unsupported = 1; }
+def : WriteRes<WriteFCopy, []>   { let Unsupported = 1; }
+def : WriteRes<WriteFImm, []>    { let Unsupported = 1; }
+def : WriteRes<WriteFMul, []>    { let Unsupported = 1; }
+def : WriteRes<WriteFDiv, []>    { let Unsupported = 1; }
+def : WriteRes<WriteVd, []>      { let Unsupported = 1; }
+def : WriteRes<WriteVq, []>      { let Unsupported = 1; }
+def : WriteRes<WriteVLD, []>     { let Unsupported = 1; }
+def : WriteRes<WriteVST, []>     { let Unsupported = 1; }
+def : WriteRes<WriteSys, []>     { let Unsupported = 1; }
+def : WriteRes<WriteBarrier, []> { let Unsupported = 1; }
+def : WriteRes<WriteHint, []>    { let Unsupported = 1; }
+def : WriteRes<WriteLDHi, []>    { let Unsupported = 1; }
+def : WriteRes<WriteAtomic, []>  { let Unsupported = 1; }
+
+// These ReadAdvance entries will be defined in later implementation
+def : ReadAdvance<ReadI,       0>;
+def : ReadAdvance<ReadISReg,   0>;
+def : ReadAdvance<ReadIEReg,   0>;
+def : ReadAdvance<ReadIM,      0>;
+def : ReadAdvance<ReadIMA,     0>;
+def : ReadAdvance<ReadID,      0>;
+def : ReadAdvance<ReadExtrHi,  0>;
+def : ReadAdvance<ReadAdrBase, 0>;
+def : ReadAdvance<ReadVLD,     0>;
+def : ReadAdvance<ReadST,      0>;
+
+
+//IXU resource definition
+// 1 cycles NO pipe
+def ORYONWrite_1Cyc_NONE : SchedWriteRes<[]>;
+
+// 1 cycles on I01.
+def ORYONWrite_1Cyc_I01 : SchedWriteRes<[ORYONI01]>;
+
+def ORYONWrite_1Cyc_2Uops_I01 : SchedWriteRes<[ORYONI01]> {
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_1Cyc_I0 : SchedWriteRes<[ORYONI0]>;
+
+// 7 cycles on I2. PAC*/AUT* instructions
+def ORYONWrite_7Cyc_I2 : SchedWriteRes<[ORYONI2]> {
+  let Latency = 7;
+}
+
+// 7 cycles on I2. PAC*/AUT* instructions
+def ORYONWrite_7Cyc_3Uops_I2 : SchedWriteRes<[ORYONI2]> {
+  let Latency = 7;
+  let NumMicroOps = 3;
+}
+
+// 9 (7+1+1) cycles on I2 and I0/I1, I0. Authentication branch instructions
+// these instructions are broken down to three uops
+// a.	PtrAuth on pipe 2 taking 7 cycles
+// b.	Link Register Update on pipes 0 and 1 taking 1 cycle
+// c.	Indirect branch on pipe 0 taking 1 cycle
+
+def ORYONWrite_9Cyc_I012 : SchedWriteRes<[ORYONI2, ORYONI01]> {
+  let Latency = 9;
+  let NumMicroOps = 3;
+}
+
+// 3 cycles on I2. CRC32 and CRC32C instructions
+def ORYONWrite_3Cyc_I2 : SchedWriteRes<[ORYONI2]> {
+  let Latency = 3;
+}
+
+// 1 cycle on I012345
+def ORYONWrite_1Cyc_I012345 : SchedWriteRes<[ORYONI012345]>;
+
+// 1 cycle on I0123
+def ORYONWrite_1Cyc_I0123 : SchedWriteRes<[ORYONI0123]>;
+
+// 1 cycle on 2 of I012345
+def ORYONWrite_1Cyc_I012345_I012345 :
+SchedWriteRes<[ORYONI012345, ORYONI012345]> ;
+
+// 2 cycle on 2 of I0123 with ReleaseAtCycles
+def ORYONWrite_2Cyc_I0123_I0123_RC :
+SchedWriteRes<[ORYONI0123, ORYONI0123]> {
+  let Latency = 2;
+  let ReleaseAtCycles = [2,2];
+}
+
+// 2 cycle on 2 of I012345
+def ORYONWrite_2Cyc_I012345_I012345_RC :
+SchedWriteRes<[ORYONI012345, ORYONI012345]> {
+  let Latency = 2;
+  let ReleaseAtCycles = [2,2];
+}
+
+// 3 cycle on 2 of I45
+def ORYONWrite_3Cyc_I45_I45_RC :
+SchedWriteRes<[ORYONI45, ORYONI45]> {
+  let Latency = 3;
+  let ReleaseAtCycles = [2,2];
+}
+
+// 3 cycle on I45
+def ORYONWrite_3Cyc_I45 : SchedWriteRes<[ORYONI45]> {
+  let Latency = 3;
+}
+
+// 7 cycle on I2 32-bit integer division
+def ORYONWrite_7Cyc_I2_RC : SchedWriteRes<[ORYONI2]> {
+  let Latency = 7;
+  let ReleaseAtCycles = [2];
+}
+
+// 9 cycle on I2 64-bit integer division
+def ORYONWrite_9Cyc_I2_RC : SchedWriteRes<[ORYONI2]> {
+  let Latency = 9;
+  let ReleaseAtCycles = [2];
+}
+
+// LSU resource definition
+// need to define WriteLDAdr, WriteAdrAdr, WriteLDHi, WriteSTX
+// 4 cycle on LS(P6789)
+def ORYONWrite_4Cyc_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 4;
+}
+
+// 4 cycle for Post/Pre inc/dec access, also covers all pair loads Post/Pre
+def ORYONWrite_4Cyc_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 4;
+}
+
+// 5 (4+1) for VXU SIMD access/could also include FP
+// resource might not be correct, as VXU resource not included
+def ORYONWrite_5Cyc_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+}
+
+def ORYONWrite_5Cyc_2Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_5Cyc_3Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_5Cyc_4Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 4;
+}
+
+def ORYONWrite_5Cyc_5Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 5;
+}
+
+def ORYONWrite_5Cyc_6Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 6;
+}
+
+def ORYONWrite_5Cyc_8Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 8;
+}
+
+def ORYONWrite_5Cyc_10Uops_LD : SchedWriteRes<[ORYONLD]> {
+  let Latency = 5;
+  let NumMicroOps = 10;
+}
+
+// 6 cycle for Post/Pre inc/dec access
+def ORYONWrite_5Cyc_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+}
+
+def ORYONWrite_5Cyc_2Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_5Cyc_3Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_5Cyc_4Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 4;
+}
+
+def ORYONWrite_5Cyc_5Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 5;
+}
+
+def ORYONWrite_5Cyc_6Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 6;
+}
+
+def ORYONWrite_5Cyc_8Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 8;
+}
+
+def ORYONWrite_5Cyc_10Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> {
+  let Latency = 5;
+  let NumMicroOps = 10;
+}
+
+// 1 cycle for all generic stores
+def ORYONWrite_1Cyc_ST : SchedWriteRes<[ORYONST]>;
+
+def ORYONWrite_1Cyc_2Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_1Cyc_3Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_1Cyc_4Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 4;
+}
+
+def ORYONWrite_1Cyc_5Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 5;
+}
+
+def ORYONWrite_1Cyc_6Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 6;
+}
+
+def ORYONWrite_1Cyc_8Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 8;
+}
+
+def ORYONWrite_1Cyc_10Uops_ST : SchedWriteRes<[ORYONST]> {
+  let NumMicroOps = 10;
+}
+
+// 1 cycle for neon write: float + ASIMD with Post/Pre Inc/Dec access
+// also includes Pair store until further informed
+def ORYONWrite_1Cyc_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_1Cyc_2Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_1Cyc_3Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_1Cyc_4Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 4;
+}
+
+def ORYONWrite_1Cyc_5Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 5;
+}
+
+def ORYONWrite_1Cyc_6Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 6;
+}
+
+def ORYONWrite_1Cyc_8Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 8;
+}
+
+def ORYONWrite_1Cyc_10Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> {
+  let NumMicroOps = 10;
+}
+
+// VXU resource definition
+
+// I2V instruction has 1 uOp
+// I2v with convert has 2 uOps
+// all I2V, V2I's throughputs are 2
+// On VXU doc, p37 -- latencies and throughput
+// P41, resource taken, P42, uOps
+def ORYONWrite_I2V_4Cyc_I45 : SchedWriteRes<[ORYONI2V]> {
+  let Latency = 4;
+}
+
+// inline a FCVT, so add one more uOp
+def ORYONWrite_I2V_7Cyc_I45 : SchedWriteRes<[ORYONI2V]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+
+// V2I move instruction has 1/2 uOps, P42 in VXU doc
+// Latency is 3, FCVT is also 3 cycle
+// move + convert is 6 (3+3) cycles
+// throughput is 2
+def ORYONWrite_V2I_3Cyc_FP01 : SchedWriteRes<[ORYONV2I]> {
+  let Latency = 3;
+}
+
+// inline a FCVT, so add one more uOp
+def ORYONWrite_V2I_6Cyc_FP01 : SchedWriteRes<[ORYONV2I]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_V2V_2Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 2;
+}
+
+def ORYONWrite_V2V_3Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 3;
+}
+
+def ORYONWrite_V2V_6Cyc_FP01 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 6;
+  let NumMicroOps = 3;
+}
+
+def ORYONWrite_4Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 4;
+}
+
+def ORYONWrite_3Cyc_FP0 : SchedWriteRes<[ORYONFP0]> {
+  let Latency = 3;
+}
+
+def ORYONWrite_3Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 3;
+}
+
+def ORYONWrite_3Cyc_2Uops_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+
+def ORYONWrite_2Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 2;
+}
+
+def ORYONWrite_2Cyc_FP01 : SchedWriteRes<[ORYONFP01]> {
+  let Latency = 2;
+}
+
+// 2 cycle on FP1
+def ORYONWrite_2Cyc_FP1 : SchedWriteRes<[ORYONFP1]> {
+  let Latency = 2;
+}
+
+// 3 cycle on FP1
+def ORYONWrite_3Cyc_FP1 : SchedWriteRes<[ORYONFP1]> {
+  let Latency = 3;
+}
+
+// 4 cycle , 0.5 throughput on FP1
+def ORYONWrite_4Cyc_FP1_RC4 : SchedWriteRes<[ORYONFP1]> {
+  let Latency = 4;
+  let ReleaseAtCycles = [4];
+}
+
+// 5 cycle , 1 throughput on FP1
+def ORYONWrite_5Cyc_FP1 : SchedWriteRes<[ORYONFP1]> {
+  let Latency = 5;
+}
+
+// 8 cycle , 2 throughput on FP0123
+def ORYONWrite_8Cyc_FP0123_RC : SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 8;
+  let ReleaseAtCycles = [2];
+}
+
+def ORYONWrite_6Cyc_FP3 : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 6;
+}
+
+def ORYONWrite_7Cyc_FP3 : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 7;
+}
+
+def ORYONWrite_8Cyc_FP3 : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 8;
+}
+
+def ORYONWrite_9Cyc_FP3 : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 9;
+}
+
+def ORYONWrite_10Cyc_FP3 : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 10;
+}
+
+def ORYONWrite_8Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 8;
+  let ReleaseAtCycles = [2];
+}
+
+def ORYONWrite_10Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 10;
+  let ReleaseAtCycles = [2];
+}
+
+def ORYONWrite_13Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> {
+  let Latency = 13;
+  let ReleaseAtCycles = [2];
+}
+
+def ORYONWrite_4Cyc_FP0123_RC :
+SchedWriteRes<[ORYONFP0123]> {
+  let Latency = 4;
+  let ReleaseAtCycles = [2];
+}
+
+def ORYONWrite_4Cyc_FP0123_FP0123_RC :
+SchedWriteRes<[ORYONFP0123, ORYONFP0123]> {
+  let Latency = 4;
+  let NumMicroOps = 2;
+  let ReleaseAtCycles = [2,2];
+}
+
+def ORYONWrite_4Cyc_FP0123_FP0123_FP0123_RC :
+SchedWriteRes<[ORYONFP0123, ORYONFP0123, ORYONFP0123]> {
+  let Latency = 4;
+  let NumMicroOps = 3;
+  let ReleaseAtCycles = [3,3,3];
+}
+
+def ORYONWrite_6Cyc_FP0123_FP0123_FP0123_FP0123_RC :
+SchedWriteRes<[ORYONFP0123, ORYONFP0123, ORYONFP0123, ORYONFP0123]> {
+  let Latency = 6;
+  let NumMicroOps = 4;
+  let ReleaseAtCycles = [6,6,6,6];
+}
+
+//===----------------------------------------------------------------------===//
+// Instruction Tables in IXU
+//===----------------------------------------------------------------------===//
+
+//---
+// Arithmetic Instructions
+//---
+
+//1, 1, 6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+            (instregex "^ADD(W|X)r(i|r|x)", "^SUB(W|X)r(i|r|x)")>;
+
+//2,2,3
+def : InstRW<[ORYONWrite_2Cyc_I012345_I012345_RC],
+            (instregex "^ADD(W|X)rs", "^SUB(W|X)rs")>;
+
+//1,1,4 alias CMP, CMN on page 75
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^ADDS(W|X)r(i|r|x)(64)?", "^SUBS(W|X)r(i|r|x)")>;
+
+//2,2,2 alias CMP, CMN on page 75
+def : InstRW<[ORYONWrite_2Cyc_I0123_I0123_RC],
+            (instregex "^ADDS(W|X)rs", "^SUBS(W|X)rs")>;
+
+//1,1,4
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^ADC(W|X)r","^SBC(W|X)r",
+                       "^ADCS(W|X)r","^SBCS(W|X)r")>;
+
+//1,1,2
+def : InstRW<[ORYONWrite_1Cyc_2Uops_I01],
+            (instrs ADR,ADRP)>;
+
+//1,1,4
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^CSEL(W|X)r", "^CSINV(W|X)r",
+                       "^CSNEG(W|X)r", "^CSINC(W|X)r")>;
+
+//---
+//Compare Instruciton
+//---
+
+// We have CCMP, CCMN as LLVM DAG node
+// CMP is an alias of SUBS as above
+// CMN is an alias of ADDS as above
+// We also have no way to get shift compare node in LLVM
+//2,2,1.5 CMP, CMN
+
+//1,1,4
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^CCMP(W|X)(i|r)", "^CCMN(W|X)(i|r)")>;
+
+//---
+// Branch
+//---
+
+def : InstRW<[ORYONWrite_1Cyc_NONE], (instrs B)>;
+def : InstRW<[ORYONWrite_1Cyc_I01], (instrs BL)>;
+def : InstRW<[ORYONWrite_1Cyc_I01],
+            (instrs Bcc, CBZW, CBZX, CBNZW, CBNZX, TBZW, TBZX, TBNZW, TBNZX)>;
+def : InstRW<[ORYONWrite_1Cyc_I0], (instrs BR, BLR)>;
+def : InstRW<[ORYONWrite_1Cyc_I0], (instrs RET)>;
+
+// 3 uOp, 1 cycle for branch, 7 cycle for Authentication,
+// 1 cycle for updating link register
+// V8.3a PAC
+def : InstRW<[ORYONWrite_9Cyc_I012],
+            (instrs BLRAA, BLRAAZ, BLRAB, BLRABZ,
+                    BRAA, BRAAZ, BRAB, BRABZ)>;
+def : InstRW<[ORYONWrite_9Cyc_I012], (instrs RETAA, RETAB, ERETAA, ERETAB)>;
+
+def : InstRW<[ORYONWrite_7Cyc_3Uops_I2], (instregex "^LDRAA", "^LDRAB")>;
+
+// Logical Instructions
+//---
+
+//1,1,4 TST is an alias of ANDS
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^ANDS(W|X)r(i|r|x)", "^BICS(W|X)r(i|r|x)")>;
+
+//2,2,2 TST shift is an alias
+def : InstRW<[ORYONWrite_2Cyc_I0123_I0123_RC],
+            (instregex "^ANDS(W|X)rs", "^BICS(W|X)rs")>;
+
+//1,1,6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+            (instregex "^AND(W|X)r(i|r|x)", "^EOR(W|X)r(i|r|x)",
+                       "^ORR(W|X)r(i|r|x)", "^BIC(W|X)r(i|r|x)",
+                       "^EON(W|X)r(i|r|x)", "^ORN(W|X)r(i|r|x)")>;
+
+//2,2,3
+def : InstRW<[ORYONWrite_2Cyc_I012345_I012345_RC],
+            (instregex "^AND(W|X)rs", "^EOR(W|X)rs", "^ORR(W|X)rs",
+                       "^BIC(W|X)rs", "^EON(W|X)rs", "^ORN(W|X)rs")>;
+
+
+//---
+// Shift Instructions
+//---
+
+//1,1,6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+            (instregex "^ASRV(W|X)r", "^LSLV(W|X)r",
+                       "^LSRV(W|X)r", "^RORV(W|X)r",
+                       "RMIF")>;
+
+//---
+// Move-Data Bit-field and Sign_Extension Instructions
+//---
+
+//1,1,6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+            (instregex "^MOVK(W|X)i", "^MOVN(W|X)i",
+                       "^MOVZ(W|X)i", "^SBFM(W|X)ri",
+                       "^UBFM(W|X)ri", "^BFM(W|X)ri",
+                       "^SXT(W|B|H|X)", "^UXT(H|B)")>;
+
+// COPY instruction is an LLVM internal DAG node, needs further study
+def : InstRW<[ORYONWrite_1Cyc_I012345], (instrs COPY)>;
+
+//---
+// Reverse Instructions
+//---
+
+//1,1,6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+            (instregex "^RBIT(W|X)r", "^REV(16|32|64)?(W|X)r")>;
+
+
+//---
+// Flag Manipulate Instructions
+//---
+
+//1,1,4
+def : InstRW<[ORYONWrite_1Cyc_I0123],
+            (instregex "^SETF8", "^SETF16", "^CFINV")>;
+
+//---
+// Miscellaneous Instructions
+//---
+
+//1,1,6
+def : InstRW<[ORYONWrite_1Cyc_I012345],
+              (instregex "^CLS(W|X)r$", "^CLZ(W|X)r$", "^EXTR(W|X)rri")>;
+
+
+//---
+// Multiply Instructions
+//---
+
+//1,3,2
+def : InstRW<[ORYONWrite_3Cyc_I45],
+            (instregex "^MADD(W|X)rrr", "^MSUB(W|X)rrr",
+                       "^(S|U)MADDLrrr", "^(S|U)MSUBLrrr",
+                       "^(S|U)MULHrr")>;
+
+//---
+// Divide Instructions
+//---
+
+def : InstRW<[ORYONWrite_7Cyc_I2_RC],
+             (instregex "^(S|U)DIVWr")>;
+
+def : InstRW<[ORYONWrite_9Cyc_I2_RC],
+             (instregex "^(S|U)DIVXr")>;
+
+
+//---
+// Cryptgraphy Instructions
+//
+//1,3,1  on I2
+def : InstRW<[ORYONWrite_3Cyc_I2],
+            (instregex "^CRC32(B|H|W|X)rr", "^CRC32C(B|H|W|X)rr")>;
+
+//---
+// PAU instructions
+//---
+
+// on p47 of IXU document, we have 7 cycles for all PAU instructions
+// here we just assume all signing and pauth instructions are 7 cycles
+// assume all are 7 cycles here
+
+// signing instrucitons
+def : InstRW<[ORYONWrite_7Cyc_I2], (instrs PACIA, PACIB,
+                                            PACDA, PACDB,
+                                            PACIZA, PACIZB,
+                                            PACDZA, PACDZB,
+                                            PACGA)>;
+// authentication instrucitons
+def : InstRW<[ORYONWrite_7Cyc_I2], (instrs AUTIA, AUTIB,
+                                            AUTDA, AUTDB,
+                                            AUTIZA, AUTIZB,
+                                            AUTDZA, AUTDZB)>;
+def : InstRW<[ORYONWrite_7Cyc_I2], (instrs XPACI, XPACD)>;
+
+//===----------------------------------------------------------------------===//
+// Instruction Tables in LSU
+//===----------------------------------------------------------------------===//
+
+// 4 cycle Load-to-use from L1D$
+// Neon load with 5 cycle
+// 6 cycle to STA ?
+// STD cycle ?
+// NEON STD + 2
+
+// Load Instructions
+// FP Load Instructions
+
+// Load pair, immed pre-index, normal
+// Load pair, immed pre-index, signed words
+// Load pair, immed post-index, normal
+// Load pair, immed post-index, signed words
+// NOTE: Handled by WriteLD, WriteLDHi, WriteAdr.
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPDi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPQi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPSi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPXi)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPDi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPQi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPSi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPSWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPXi)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBui)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDui)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHui)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQui)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSui)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDl)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQl)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWl)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXl)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRBi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRHi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRXi)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSBWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSBXi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSHWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSHXi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSWi)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPDpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPQpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPSpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPWpre)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRDpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRQpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRWpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRXpre)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBWpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBXpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBWpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBXpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHWpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHXpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHWpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHXpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBBpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBBpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHHpre)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHHpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPDpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPQpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPSpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPWpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345],
+            (instrs LDPXpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRDpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRQpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRWpost)>;
+def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRXpost)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHHroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHWroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHXroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWroW)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXroW)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHHroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHWroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHXroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWroX)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXroX)>;
+
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURBi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURBBi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURDi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURHi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURHHi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURQi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURXi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSBWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSBXi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSHWi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSHXi)>;
+def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSWi)>;
+
+
+
+// Store register, immed post-index
+// NOTE: Handled by WriteST, ReadAdrBase
+
+// Store register, immed pre-index
+// NOTE: Handled by WriteST
+
+// Store pair, immed post-index, W-form
+// Store pair, immed post-indx, X-form
+// Store pair, immed pre-index, W-form
+// Store pair, immed pre-index, X-form
+// NOTE: Handled by WriteSTP.
+
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURBi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURBBi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURDi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURHi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURHHi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURQi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURSi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURWi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURXi)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRBi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRHi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRWi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRXi)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPDi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPQi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPXi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPWi)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPDi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPQi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPXi)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPWi)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRBui)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRDui)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRHui)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRQui)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRXui)>;
+def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRWui)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STPDpre, STPDpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STPSpre, STPSpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STPWpre, STPWpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STPXpre, STPXpost)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRBpre, STRBpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRBBpre, STRBBpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRDpre, STRDpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRHpre, STRHpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRHHpre, STRHHpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRQpre, STRQpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRSpre, STRSpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRWpre, STRWpost)>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instrs STRXpre, STRXpost)>;
+
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRBroW, STRBroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRDroW, STRDroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRHroW, STRHroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRHHroW, STRHHroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRQroW, STRQroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRSroW, STRSroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRWroW, STRWroX)>;
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instrs STRXroW, STRXroX)>;
+
+// ASIMD Load instructions, 4 cycle access + 2 cycle NEON access
+// ASIMD load, 1 element, multiple, 1 reg, D-form 1uOps
+// ASIMD load, 1 element, multiple, 1 reg, Q-form 1uOps
+def : InstRW<[ORYONWrite_5Cyc_LD],
+            (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_LD_I012345],
+            (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 1 element, multiple, 2 reg, D-form 3 uOps
+// ASIMD load, 1 element, multiple, 2 reg, Q-form 2 uOps
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD],
+            (instregex "^LD1Twov(8b|4h|2s|1d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD],
+            (instregex "^LD1Twov(16b|8h|4s|2d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345],
+            (instregex "^LD1Twov(8b|4h|2s|1d)_POST$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345],
+            (instregex "^LD1Twov(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 1 element, multiple, 3 reg, D-form 4 uOps
+// ASIMD load, 1 element, multiple, 3 reg, Q-form 3 uOps
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD],
+            (instregex "^LD1Threev(8b|4h|2s|1d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD],
+            (instregex "^LD1Threev(16b|8h|4s|2d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345],
+            (instregex "^LD1Threev(8b|4h|2s|1d)_POST$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345],
+            (instregex "^LD1Threev(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 1 element, multiple, 4 reg, D-form 6 uOps
+// ASIMD load, 1 element, multiple, 4 reg, Q-form 4 uOps
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD],
+            (instregex "^LD1Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD],
+            (instregex "^LD1Fourv(16b|8h|4s|2d)$")>;
+
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345],
+            (instregex "^LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345],
+            (instregex "^LD1Fourv(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 1 element, one lane, B/H/S 2uOps
+// ASIMD load, 1 element, one lane, D     2UOps
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD], (instregex "^LD1i(8|16|32|64)$")>;
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345],
+            (instregex "^LD1i(8|16|32|64)_POST$")>;
+
+// ASIMD load, 1 element, all lanes, D-form, B/H/S 2uOps
+// ASIMD load, 1 element, all lanes, D-form, D     2uOps
+// ASIMD load, 1 element, all lanes, Q-form        2uOps
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD],
+            (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345],
+            (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 2 element, multiple, D-form, B/H/S 3 uOps
+// ASIMD load, 2 element, multiple, Q-form, D     4 uOps
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD],
+            (instregex "^LD2Twov(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD],
+            (instregex "^LD2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345],
+            (instregex "^LD2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345],
+            (instregex "^LD2Twov(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 2 element, one lane, B/H           3 uOps
+// ASIMD load, 2 element, one lane, S             3 uOps
+// ASIMD load, 2 element, one lane, D             3 uOps
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], (instregex "^LD2i(8|16|32|64)$")>;
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345],
+            (instregex "^LD2i(8|16|32|64)_POST$")>;
+
+// ASIMD load, 2 element, all lanes, D-form, B/H/S 3 uOps
+// ASIMD load, 2 element, all lanes, D-form, D     3 uOps
+// ASIMD load, 2 element, all lanes, Q-form        3 uOps
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD],
+            (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345],
+            (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 3 element, multiple, D-form, B/H/S  5 uOps
+// ASIMD load, 3 element, multiple, Q-form, B/H/S  6 uOps
+// ASIMD load, 3 element, multiple, Q-form, D      6 uOps
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD],
+            (instregex "^LD3Threev(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD],
+            (instregex "^LD3Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345],
+            (instregex "^LD3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345],
+            (instregex "^LD3Threev(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD load, 3 element, one lone, B/H            4 uOps
+// ASIMD load, 3 element, one lane, S              4 uOps
+// ASIMD load, 3 element, one lane, D              5 uOps
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], (instregex "^LD3i(8|16|32)$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], (instregex "^LD3i(64)$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345],
+            (instregex "^LD3i(8|16|32)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345],
+            (instregex "^LD3i(64)_POST$")>;
+
+// ASIMD load, 3 element, all lanes, D-form, B/H/S 4 uOps
+// ASIMD load, 3 element, all lanes, D-form, D     5 uOps
+// ASIMD load, 3 element, all lanes, Q-form, B/H/S 4 uOps
+// ASIMD load, 3 element, all lanes, Q-form, D     5 uOps
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD],
+            (instregex "^LD3Rv(8b|4h|2s|16b|8h|4s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD],
+            (instregex "^LD3Rv(1d|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345],
+            (instregex "^LD3Rv(8b|4h|2s|16b|8h|4s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345],
+            (instregex "^LD3Rv(1d|2d)_POST$")>;
+
+// ASIMD load, 4 element, multiple, D-form, B/H/S  6 uOps
+// ASIMD load, 4 element, multiple, Q-form, B/H/S  10 uOps
+// ASIMD load, 4 element, multiple, Q-form, D      8 uOps
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD],
+            (instregex "^LD4Fourv(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_10Uops_LD],
+            (instregex "^LD4Fourv(16b|8h|4s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_8Uops_LD],
+            (instregex "^LD4Fourv(2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345],
+            (instregex "^LD4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_10Uops_LD_I012345],
+            (instregex "^LD4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_8Uops_LD_I012345],
+            (instregex "^LD4Fourv(2d)_POST$")>;
+
+// ASIMD load, 4 element, one lane, B/H            5 uOps
+// ASIMD load, 4 element, one lane, S              5 uOps
+// ASIMD load, 4 element, one lane, D              6 uOps
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], (instregex "^LD4i(8|16|32)$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], (instregex "^LD4i(64)$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345],
+            (instregex "^LD4i(8|16|32)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345],
+            (instregex "^LD4i(64)_POST$")>;
+
+// ASIMD load, 4 element, all lanes, D-form, B/H/S    5 uOps
+// ASIMD load, 4 element, all lanes, D-form, D        6 uOps
+// ASIMD load, 4 element, all lanes, Q-form, B/H/S    5 uOps
+// ASIMD load, 4 element, all lanes, Q-form, D        6 uOps
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD],
+            (instregex "^LD4Rv(8b|4h|2s|16b|8h|4s)$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD],
+            (instregex "^LD4Rv(1d|2d)$")>;
+def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345],
+            (instregex "^LD4Rv(8b|4h|2s|16b|8h|4s)_POST$")>;
+def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345],
+            (instregex "^LD4Rv(1d|2d)_POST$")>;
+
+// ASIMD Store Instructions
+// ASIMD store, 1 element, multiple, 1 reg, D-form    1 uOps
+// ASIMD store, 1 element, multiple, 1 reg, Q-form    1 uops
+def : InstRW<[ORYONWrite_1Cyc_ST],
+            (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_ST_I012345],
+            (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 1 element, multiple, 2 reg, D-form    2 uOps
+// ASIMD store, 1 element, multiple, 2 reg, Q-form    2 uOps
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST],
+            (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345],
+            (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 1 element, multiple, 3 reg, D-form    3 uOps
+// ASIMD store, 1 element, multiple, 3 reg, Q-form    3 uOps
+def : InstRW<[ORYONWrite_1Cyc_3Uops_ST],
+            (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_3Uops_ST_I012345],
+            (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 1 element, multiple, 4 reg, D-form    4 uOps
+// ASIMD store, 1 element, multiple, 4 reg, Q-form    4 uOps
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST],
+            (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345],
+            (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 1 element, one lane, B/H/S            2 uOps
+// ASIMD store, 1 element, one lane, D                2 uOps
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST],
+            (instregex "^ST1i(8|16|32|64)$")>;
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345],
+            (instregex "^ST1i(8|16|32|64)_POST$")>;
+
+// ASIMD store, 2 element, multiple, D-form, B/H/S    2 uOps
+// ASIMD store, 2 element, multiple, Q-form, B/H/S    4 uOps
+// ASIMD store, 2 element, multiple, Q-form, D        4 uOps
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST],
+            (instregex "^ST2Twov(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST],
+            (instregex "^ST2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345],
+            (instregex "^ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345],
+            (instregex "^ST2Twov(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 2 element, one lane, B/H/S            2 uOps
+// ASIMD store, 2 element, one lane, D                2 uOps
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST],
+            (instregex "^ST2i(8|16|32|64)$")>;
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345],
+            (instregex "^ST2i(8|16|32|64)_POST$")>;
+
+// ASIMD store, 3 element, multiple, D-form, B/H/S    4 uOps
+// ASIMD store, 3 element, multiple, Q-form, B/H/S    6 uOps
+// ASIMD store, 3 element, multiple, Q-form, D        6 uOps
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST],
+            (instregex "^ST3Threev(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_1Cyc_6Uops_ST],
+            (instregex "^ST3Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345],
+            (instregex "^ST3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_6Uops_ST_I012345],
+            (instregex "^ST3Threev(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store, 3 element, one lane, B/H              2 uOps
+// ASIMD store, 3 element, one lane, S                2 uOps
+// ASIMD store, 3 element, one lane, D                4 uOps
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], (instregex "^ST3i(8|16|32)$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], (instregex "^ST3i(64)$")>;
+def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345],
+            (instregex "^ST3i(8|16|32)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345],
+            (instregex "^ST3i(64)_POST$")>;
+
+
+// ASIMD store, 4 element, multiple, D-form, B/H/S    5 uOps
+// ASIMD store, 4 element, multiple, Q-form, B/H/S    10 uOps
+// ASIMD store, 4 element, multiple, Q-form, D        8 uOps
+def : InstRW<[ORYONWrite_1Cyc_5Uops_ST],
+            (instregex "^ST4Fourv(8b|4h|2s)$")>;
+def : InstRW<[ORYONWrite_1Cyc_10Uops_ST],
+            (instregex "^ST4Fourv(16b|8h|4s)$")>;
+def : InstRW<[ORYONWrite_1Cyc_8Uops_ST],
+            (instregex "^ST4Fourv(2d)$")>;
+def : InstRW<[ORYONWrite_1Cyc_5Uops_ST_I012345],
+            (instregex "^ST4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_10Uops_ST_I012345],
+            (instregex "^ST4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_8Uops_ST_I012345],
+            (instregex "^ST4Fourv(2d)_POST$")>;
+
+// ASIMD store, 4 element, one lane, B/H              3 uOps
+// ASIMD store, 4 element, one lane, S                3 uOps
+// ASIMD store, 4 element, one lane, D                4 uOps
+def : InstRW<[ORYONWrite_1Cyc_3Uops_ST], (instregex "^ST4i(8|16|32)$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], (instregex "^ST4i(64)$")>;
+def : InstRW<[ORYONWrite_1Cyc_3Uops_ST_I012345],
+            (instregex "^ST4i(8|16|32)_POST$")>;
+def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345],
+            (instregex "^ST4i(64)_POST$")>;
+
+
+//===----------------------------------------------------------------------===//
+// Instruction Tables in VXU
+//===----------------------------------------------------------------------===//
+// all uOps are not clearly written in the VXU document
+
+// I2V
+def : InstRW<[ORYONWrite_I2V_4Cyc_I45], (instregex "^FMOV[HSD][WX]r", "^FMOVDXHighr")>;
+
+// I2V with convert
+def : InstRW<[ORYONWrite_I2V_7Cyc_I45], (instregex "^[SU]CVTF[SU][XW][HSD]ri")>;
+
+// V2I
+def : InstRW<[ORYONWrite_V2I_3Cyc_FP01], (instregex "^FMOV[WX][HSD]r", "FMOVXDHighr")>;
+
+// V2I with convert 2nd [SU] necessary?
+def : InstRW<[ORYONWrite_V2I_6Cyc_FP01], (instregex "^FCVT[AMNPZ][SU][SU][XW][HSD]r")>;
+
+// float to float move immediate, row 7 in big chart
+def : InstRW<[ORYONWrite_V2V_2Cyc_FP0123], (instregex "^FMOV[HSD]r")>;
+def : InstRW<[ORYONWrite_V2V_2Cyc_FP0123], (instregex "^FMOV[HSD]i")>;
+
+// float to float conversion within VXU, precision conversion
+def : InstRW<[ORYONWrite_V2V_6Cyc_FP01], (instregex "^FJCVTZS")>;
+def : InstRW<[ORYONWrite_V2V_3Cyc_FP0123], (instregex "^FCVT[HSD][HSD]r",
+                                                       "^FRINT(A|I|M|N|P|X|Z)(Sr|Dr)")>;
+
+// floating comparison write to NZCV
+def : InstRW<[ORYONWrite_2Cyc_FP01], (instregex "^FCMP(E)?[HSD]r[ir]")>;
+def : InstRW<[ORYONWrite_2Cyc_FP01], (instregex "^FCCMP(E)?[HSD]rr")>;
+
+// floating point conditional select
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^FCSEL")>;
+
+// floating multiply-add
+def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^(F|FN)MADD", "^(F|FN)MSUB")>;
+
+// floating unary, cycle/throughput? xls row14
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^F(ABS|NEG)[SD]r")>;
+
+//floating division/square root
+def : InstRW<[ORYONWrite_7Cyc_FP3], (instregex "^FDIVHrr")>;
+def : InstRW<[ORYONWrite_8Cyc_FP3], (instregex "^FDIVSrr")>;
+def : InstRW<[ORYONWrite_10Cyc_FP3], (instregex "^FDIVDrr")>;
+
+def : InstRW<[ORYONWrite_8Cyc_FP3_RC], (instregex "^FSQRTHr")>;
+def : InstRW<[ORYONWrite_10Cyc_FP3_RC], (instregex "^FSQRTSr")>;
+def : InstRW<[ORYONWrite_13Cyc_FP3_RC], (instregex "^FSQRTDr")>;
+
+//==========
+// SIMD move instructions
+//==========
+
+// ASIMD DUP element
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^DUPv.+lane")>;
+// ASIMD DUP general thoughput undecided, 3? FP0123
+// VXU doc, p42, 2 uOps
+def : InstRW<[ORYONWrite_3Cyc_2Uops_FP0123], (instregex "^DUPv.+gpr")>;
+
+// ASIMD insert, element to element
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^INSv.+lane")>;
+// ASIMD insert,  gen reg 3? FP0123?
+def : InstRW<[ORYONWrite_3Cyc_2Uops_FP0123], (instregex "^INSv.+gpr")>;
+
+// ASIMD move, FP immed
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^FMOVv")>;
+
+// ASIMD transfer, element to gen reg
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^[SU]MOVv")>;
+
+//==========
+// SIMD arithmetic instructions
+//==========
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDv", "^SUBv",
+                                         "^BIFv", "^BITv", "^BSLv",
+                                         "^ANDv", "^BICv", "^EORv",
+                                         "^ORRv", "^ORNv")>;
+
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^FABDv", "^FADDv", "^FSUBv")>;
+
+// floating division
+def : InstRW<[ORYONWrite_6Cyc_FP3], (instregex "^FDIVv.*16$")>;
+def : InstRW<[ORYONWrite_7Cyc_FP3], (instregex "^FDIVv.*32$")>;
+def : InstRW<[ORYONWrite_9Cyc_FP3], (instregex "^FDIVv.*64$")>;
+
+def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FMUL(X)?v",
+                                                   "^FRECPSv", "^FRSQRTSv")>;
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^MLAv","^MLSv", "^MULv",
+                                                   "^PMULv", "UABAv")>;
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "SABAv", "SABDv",
+                                                   "^(SH|UH)(ADD|SUB)v",
+                                                   "^S(MAX|MIN)v",
+                                                   "^(SQ|UQ)(ADD|SUB)v",
+                                                   "^(SQ|SQR|UQ|UQR)SHLv",
+                                                   "^(SR|UR)HADDv",
+                                                   "^(SR|UR)SHLv",
+                                                   "^UABDv",
+                                                   "^U(MAX|MIN)v")>;
+// IMAX or UMAX in the above line
+//==========
+// SIMD compare instructions
+//==========
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^CMEQv","^CMGEv","^CMGTv",
+                                                   "^CMLEv","^CMLTv", "^CMHIv",
+                                                   "^CMHSv",
+                                                   "^FCMEQv", "^FCMGEv",
+                                                   "^FCMGTv", "^FCMLEv",
+                                                   "^FCMLTv",
+                                                   "^FACGEv", "^FACGTv")>;
+
+//==========
+// SIMD widening and narrowing arithmetic instructions
+//==========
+// NO need to list ADDHN2, RADDHN2, RSUBHN2 as they are not distinguished
+// from ADDHN, RADDHN, RSUBHN in td file(v16i8, v8i16, v4i32).
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDHNv",
+                                                   "^SUBHNv",
+                                                   "^RADDHNv",
+                                                   "^RSUBHNv",
+                                                   "^SABD(L|L2)v", "^UABD(L|L2)v",
+                                                   "^(S|U)(ADD|SUB)(L|L2|W|W2)v")>;
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^PMUL(L|L2)v","^SABA(L|L2)v",
+                                                   "^(S|U|SQ)(MLA|MSL|MUL)(L|L2)v")>;
+
+//==========
+// SIMD unary arithmetic instructions
+//==========
+//^MVNv is an alias of ^NOTv
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ABSv", "^CLSv","^CLZv", "^CNTv",
+                                                   "^NEGv", "^NOTv",
+                                                   "^RBITv", "^REV(16|32|64)v",
+                                                   "^SQ(ABS|NEG)v", "^SQ(XT|XTU)(N|N2)v",
+                                                   "^(SU|US)QADDv",
+                                                   "^UQXT(N|N2)v", "^XTN2?v")>;
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^FCVT(L|L2|N|N2|XN|XN2)v",
+                                                   "^FRINT[AIMNPXZ]v",
+                                                   "^FRSQRTEv",
+                                                   "^(S|U)ADALPv",
+                                                   "^(S|U)ADDLPv")>;
+
+
+def : InstRW<[ORYONWrite_3Cyc_FP0], (instregex "^URECPEv", "^URSQRTEv",
+                                                "^FRECPEv", "^FRECPXv")>;
+
+def : InstRW<[ORYONWrite_8Cyc_FP3_RC], (instregex "^FSQRTv.*16$")>;
+def : InstRW<[ORYONWrite_10Cyc_FP3_RC], (instregex "^FSQRTv.*32$")>;
+def : InstRW<[ORYONWrite_13Cyc_FP3_RC], (instregex "^FSQRTv.*64$")>;
+
+//==========
+// SIMD binary elememt arithmetic instructions
+//==========
+
+def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FMLAv", "^FMLSv")>;
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex  "^SQDMULHv",
+                                                   "^SQRD(MLA|MLS|MUL)Hv")>;
+
+//==========
+// SIMD permute instructions
+//==========
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^EXTv", "^TRN(1|2)v",
+                                                   "^UZP(1|2)v", "^ZIP(1|2)v")>;
+
+//==========
+// SIMD immediate instructions
+//==========
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex  "^MOVIv", "^MVNIv")>;
+
+//==========
+// SIMD shift(immediate) instructions
+//==========
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^RSHR(N|N2)v", "^SHLv",
+                                                   "^(SHL|SHR)(N|N2)v",
+                                                   "^SLIv",
+                                                   "^(SQ|SQR)SHR(U)?(N|N2)v",
+                                                   "^(UQ|UQR)SHR(N|N2)v",
+                                                   "^SQSHLUv",
+                                                   "^SRIv",
+                                                   "^(S|SR|U|UR)SHRv",
+                                                   "^(S|SR|U|UR)SRAv",
+                                                   "^(S|U)SHL(L|L2)v")>;
+
+//==========
+// SIMD floating-point and integer conversion instructions
+//==========
+// same as above conversion
+
+//==========
+// SIMD reduce (acoss vector lanes) instructions
+//==========
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDVv",
+                                                   "^(FMAX|FMIN)(V|NMV)v",
+                                                   "^(S|U)ADDLVv",
+                                                   "^(S|U)(MAX|MIN)Vv")>;
+//==========
+// SIMD pairwise arithmetic instructions
+//==========
+
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDPv", "^FADDPv",
+                                                   "^(FMAX|FMIN)(NMP|P)v",
+                                                   "^(S|U)(MIN|MAX)Pv")>;
+//==========
+// SIMD dot prodcut instructions
+//==========
+
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^(U|S)DOTv")>;
+
+//==========
+// SIMD table lookup instructions
+//==========
+// TBL 1-reg/2-reg; TBX 1-reg, 1uOp, throughput=4 latency=2
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instrs TBLv8i8One, TBLv16i8One,
+                                                TBXv8i8One, TBXv16i8One,
+                                                TBLv8i8Two, TBLv16i8Two)>;
+
+// TBL 3-reg/4-reg, 3uops, throughtput=4/3=1.33 latency=4
+def : InstRW<[ORYONWrite_4Cyc_FP0123_FP0123_FP0123_RC],
+            (instrs TBLv8i8Three, TBLv16i8Three,
+                    TBLv8i8Four, TBLv16i8Four)>;
+
+
+// TBX 2-reg 2 uOps, throughput=2 latency=4
+def : InstRW<[ORYONWrite_4Cyc_FP0123_FP0123_RC], (instrs TBXv8i8Two, TBXv16i8Two)>;
+
+// TBX 3-reg/4-reg, 4uOps, throughput=1, latency=6
+def : InstRW<[ORYONWrite_6Cyc_FP0123_FP0123_FP0123_FP0123_RC],
+            (instrs TBXv8i8Three, TBXv16i8Three,
+                    TBXv8i8Four, TBXv16i8Four)>;
+
+
+//==========
+// SIMD complex number arithmetic instructions
+//==========
+
+def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FCADDv", "^FCMLAv")>;
+
+//==========
+// SIMD cryptographic instructions
+//==========
+// 3,4 on IMLA, CRYP
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^AES[DE]",
+                                                   "^SM3(TT1|TT2)(A|B)")>;
+
+// 2,4 on CRYP
+def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^AESI?MC",
+                                                   "^EOR3",
+                                                   "^RAX1",
+                                                   "^XAR",
+                                                   "^BCAX",
+                                                   "^SM3SS1",
+                                                   "^SM3PART(W1|W2)")>;
+// 5,1 on CRYP
+def : InstRW<[ORYONWrite_5Cyc_FP1], (instregex "^SM4E",
+                                                "^SM4EKEY")>;
+
+// 2,1 on CRYP
+def : InstRW<[ORYONWrite_2Cyc_FP1], (instregex "^SHA1(H|SU0|SU1)",
+                                                "^SHA256SU0",
+                                                "^SHA512(SU0|SU1)")>;
+
+// 3,1 on CRYP
+def : InstRW<[ORYONWrite_3Cyc_FP1], (instregex "^SHA256SU1",
+                                                "^SHA512(H|H2)")>;
+
+// 4,0.25 on CRYP
+def : InstRW<[ORYONWrite_4Cyc_FP1_RC4], (instregex "^SHA1(C|P|M)",
+                                                "^SHA256(H|H2)")>;
+
+//==========
+// SIMD v8.6 instructions
+//==========
+// 4,2 on IMLA
+def : InstRW<[ORYONWrite_4Cyc_FP0123_RC], (instregex "^(S|U|US)MMLA$")>;
+
+// 4,0.5 on IMLA
+def : InstRW<[ORYONWrite_8Cyc_FP0123_RC], (instregex "^BFMMLA$")>;
+
+// 4,0.5 on IMLA
+def : InstRW<[ORYONWrite_8Cyc_FP0123_RC], (instregex "^BFMLAL(B|T)")>;
+
+// 3,4
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^(US|SU)DOTv")>;
+
+// 3,1
+def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^BF(16)?DOTv")>;
+
+// 3,4
+def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^BFCVT(N|N2)?$")>;
+
+
+} // SchedModel = OryonModel
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 8bc26eeef34d967fc88e816f3866ba210dbefd2e..93ea729e25509713d9a76e2f9b141f2234953d63 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -299,6 +299,13 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
     PrefLoopAlignment = Align(64);
     MaxInterleaveFactor = 4;
     break;
+  case Oryon:
+    CacheLineSize = 64;
+    PrefFunctionAlignment = Align(16);
+    MaxInterleaveFactor = 4;
+    PrefetchDistance = 128;
+    MinPrefetchStride = 1024;
+    break;
   }
 
   if (AArch64MinimumJumpTableEntries.getNumOccurrences() > 0 || !HasMinSize)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 30f0ceaf674c61ec966e48591edd3355f40ae8f5..7de9071476e7fb19cbcf11d8db46d35c1215749e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -11,7 +11,6 @@
 
 #include "AArch64TargetMachine.h"
 #include "AArch64.h"
-#include "AArch64LoopIdiomTransform.h"
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64MachineScheduler.h"
 #include "AArch64MacroFusion.h"
@@ -52,6 +51,7 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/CFGuard.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
 #include <memory>
 #include <optional>
 #include <string>
@@ -234,7 +234,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() {
   initializeAArch64DeadRegisterDefinitionsPass(*PR);
   initializeAArch64ExpandPseudoPass(*PR);
   initializeAArch64LoadStoreOptPass(*PR);
-  initializeAArch64LoopIdiomTransformLegacyPassPass(*PR);
   initializeAArch64MIPeepholeOptPass(*PR);
   initializeAArch64SIMDInstrOptPass(*PR);
   initializeAArch64O0PreLegalizerCombinerPass(*PR);
@@ -553,12 +552,9 @@ public:
 void AArch64TargetMachine::registerPassBuilderCallbacks(
     PassBuilder &PB, bool PopulateClassToPassNames) {
 
-#define GET_PASS_REGISTRY "AArch64PassRegistry.def"
-#include "llvm/Passes/TargetPassRegistry.inc"
-
   PB.registerLateLoopOptimizationsEPCallback(
       [=](LoopPassManager &LPM, OptimizationLevel Level) {
-        LPM.addPass(AArch64LoopIdiomTransformPass());
+        LPM.addPass(LoopIdiomVectorizePass());
       });
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index 8fb68b06f137803c213fbc957b73b1df20fe4d06..e396d9204716a5326a0aeb1f5b63230b73198f9f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -14,7 +14,6 @@
 #define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETMACHINE_H
 
 #include "AArch64InstrInfo.h"
-#include "AArch64LoopIdiomTransform.h"
 #include "AArch64Subtarget.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index f49c73dc795192b619ef01e93bc4fa5df54388cd..9f5756fc7e4012e5b6149610e9d38e50f792fa88 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -58,6 +58,9 @@ static cl::opt<unsigned> InlineCallPenaltyChangeSM(
 static cl::opt<bool> EnableOrLikeSelectOpt("enable-aarch64-or-like-select",
                                            cl::init(true), cl::Hidden);
 
+static cl::opt<bool> EnableLSRCostOpt("enable-aarch64-lsr-cost-opt",
+                                      cl::init(true), cl::Hidden);
+
 namespace {
 class TailFoldingOption {
   // These bitfields will only ever be set to something non-zero in operator=,
@@ -4216,3 +4219,19 @@ bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) {
     return true;
   return BaseT::shouldTreatInstructionLikeSelect(I);
 }
+
+bool AArch64TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
+                                   const TargetTransformInfo::LSRCost &C2) {
+  // AArch64 specific here is adding the number of instructions to the
+  // comparison (though not as the first consideration, as some targets do)
+  // along with changing the priority of the base additions.
+  // TODO: Maybe a more nuanced tradeoff between instruction count
+  // and number of registers? To be investigated at a later date.
+  if (EnableLSRCostOpt)
+    return std::tie(C1.NumRegs, C1.Insns, C1.NumBaseAdds, C1.AddRecCost,
+                    C1.NumIVMuls, C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
+           std::tie(C2.NumRegs, C2.Insns, C2.NumBaseAdds, C2.AddRecCost,
+                    C2.NumIVMuls, C2.ScaleCost, C2.ImmCost, C2.SetupCost);
+
+  return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 2f44aaa3e26ab834b924efdf05bd5360ba1239c0..feec1a4289c3aeac03e8adc92c7fe0baa36e15ab 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -425,6 +425,9 @@ public:
   }
 
   std::optional<unsigned> getMinPageSize() const { return 4096; }
+
+  bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
+                     const TargetTransformInfo::LSRCost &C2);
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 8e76f6c9279e732874a2296e1239ca6dc45eba55..639bc0707dff249b8b9afe6f49c1f202660f9f05 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -65,7 +65,6 @@ add_llvm_target(AArch64CodeGen
   AArch64ISelLowering.cpp
   AArch64InstrInfo.cpp
   AArch64LoadStoreOptimizer.cpp
-  AArch64LoopIdiomTransform.cpp
   AArch64LowerHomogeneousPrologEpilog.cpp
   AArch64MachineFunctionInfo.cpp
   AArch64MachineScheduler.cpp
@@ -112,6 +111,7 @@ add_llvm_target(AArch64CodeGen
   Target
   TargetParser
   TransformUtils
+  Vectorize
 
   ADD_TO_COMPONENT
   AArch64
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 07a0473888ee59ce7224d57bbf8d562cae8db0a9..42cd43c3afa37dfd0909fd7383a6ce55d0000529 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -267,9 +267,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .libcallFor({{s64, s128}})
       .minScalarOrElt(1, MinFPScalar);
 
-  getActionDefinitionsBuilder(
-      {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10,
-       G_FEXP, G_FEXP2, G_FEXP10})
+  getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
+                               G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10})
       // We need a call for these, so we always need to scalarize.
       .scalarize(0)
       // Regardless of FP16 support, widen 16-bit elements to 32-bits.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 8e302786c746eba457735458cd11e3f25a7b9d54..d0d7a9dc172474d7db9f2c13749f93a698e1851d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1534,6 +1534,12 @@ def FeatureISAVersion11_5_1 : FeatureSet<
      FeatureVGPRSingleUseHintInsts,
      Feature1_5xVGPRs])>;
 
+def FeatureISAVersion11_5_2 : FeatureSet<
+  !listconcat(FeatureISAVersion11_Common.Features,
+    [FeatureSALUFloatInsts,
+     FeatureDPPSrc1SGPR,
+     FeatureVGPRSingleUseHintInsts])>;
+
 def FeatureISAVersion12 : FeatureSet<
   [FeatureGFX12,
    FeatureLDSBankCount32,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index cad4a3430327b29f7ff42e8c4c770d97c7b592fe..609e9752223b5391705ceb013dda32ed247efa0b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -400,6 +400,36 @@ void AMDGPUAsmPrinter::emitCommonFunctionComments(
                               false);
 }
 
+SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
+  SmallString<128> Str;
+  raw_svector_ostream OSS(Str);
+  int64_t IVal;
+  if (Value->evaluateAsAbsolute(IVal)) {
+    OSS << static_cast<uint64_t>(IVal);
+  } else {
+    Value->print(OSS, MAI);
+  }
+  return Str;
+}
+
+void AMDGPUAsmPrinter::emitCommonFunctionComments(
+    const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
+    const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
+    const AMDGPUMachineFunction *MFI) {
+  OutStreamer->emitRawComment(" codeLenInByte = " + Twine(CodeSize), false);
+  OutStreamer->emitRawComment(" NumSgprs: " + getMCExprStr(NumSGPR), false);
+  OutStreamer->emitRawComment(" NumVgprs: " + getMCExprStr(NumVGPR), false);
+  if (NumAGPR && TotalNumVGPR) {
+    OutStreamer->emitRawComment(" NumAgprs: " + getMCExprStr(NumAGPR), false);
+    OutStreamer->emitRawComment(" TotalNumVgprs: " + getMCExprStr(TotalNumVGPR),
+                                false);
+  }
+  OutStreamer->emitRawComment(" ScratchSize: " + getMCExprStr(ScratchSize),
+                              false);
+  OutStreamer->emitRawComment(" MemoryBound: " + Twine(MFI->isMemoryBound()),
+                              false);
+}
+
 uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
     const MachineFunction &MF) const {
   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
@@ -554,13 +584,10 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
     OutStreamer->emitRawComment(" Kernel info:", false);
     emitCommonFunctionComments(
-        getMCExprValue(CurrentProgramInfo.NumArchVGPR, Ctx),
-        STM.hasMAIInsts() ? getMCExprValue(CurrentProgramInfo.NumAccVGPR, Ctx)
-                          : std::optional<uint32_t>(),
-        getMCExprValue(CurrentProgramInfo.NumVGPR, Ctx),
-        getMCExprValue(CurrentProgramInfo.NumSGPR, Ctx),
-        getMCExprValue(CurrentProgramInfo.ScratchSize, Ctx),
-        getFunctionCodeSize(MF), MFI);
+        CurrentProgramInfo.NumArchVGPR,
+        STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
+        CurrentProgramInfo.NumVGPR, CurrentProgramInfo.NumSGPR,
+        CurrentProgramInfo.ScratchSize, getFunctionCodeSize(MF), MFI);
 
     OutStreamer->emitRawComment(
       " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), false);
@@ -571,43 +598,38 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
       " bytes/workgroup (compile time only)", false);
 
     OutStreamer->emitRawComment(
-        " SGPRBlocks: " +
-            Twine(getMCExprValue(CurrentProgramInfo.SGPRBlocks, Ctx)),
-        false);
+        " SGPRBlocks: " + getMCExprStr(CurrentProgramInfo.SGPRBlocks), false);
+
     OutStreamer->emitRawComment(
-        " VGPRBlocks: " +
-            Twine(getMCExprValue(CurrentProgramInfo.VGPRBlocks, Ctx)),
-        false);
+        " VGPRBlocks: " + getMCExprStr(CurrentProgramInfo.VGPRBlocks), false);
 
     OutStreamer->emitRawComment(
         " NumSGPRsForWavesPerEU: " +
-            Twine(
-                getMCExprValue(CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx)),
+            getMCExprStr(CurrentProgramInfo.NumSGPRsForWavesPerEU),
         false);
     OutStreamer->emitRawComment(
         " NumVGPRsForWavesPerEU: " +
-            Twine(
-                getMCExprValue(CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx)),
+            getMCExprStr(CurrentProgramInfo.NumVGPRsForWavesPerEU),
         false);
 
-    if (STM.hasGFX90AInsts())
+    if (STM.hasGFX90AInsts()) {
+      const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
+          CurrentProgramInfo.AccumOffset, MCConstantExpr::create(1, Ctx), Ctx);
+      AdjustedAccum = MCBinaryExpr::createMul(
+          AdjustedAccum, MCConstantExpr::create(4, Ctx), Ctx);
       OutStreamer->emitRawComment(
-          " AccumOffset: " +
-              Twine((getMCExprValue(CurrentProgramInfo.AccumOffset, Ctx) + 1) *
-                    4),
-          false);
+          " AccumOffset: " + getMCExprStr(AdjustedAccum), false);
+    }
 
     OutStreamer->emitRawComment(
-        " Occupancy: " +
-            Twine(getMCExprValue(CurrentProgramInfo.Occupancy, Ctx)),
-        false);
+        " Occupancy: " + getMCExprStr(CurrentProgramInfo.Occupancy), false);
 
     OutStreamer->emitRawComment(
       " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), false);
 
     OutStreamer->emitRawComment(
         " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
-            Twine(getMCExprValue(CurrentProgramInfo.ScratchEnable, Ctx)),
+            getMCExprStr(CurrentProgramInfo.ScratchEnable),
         false);
     OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:USER_SGPR: " +
                                     Twine(CurrentProgramInfo.UserSGPR),
@@ -628,20 +650,25 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
                                     Twine(CurrentProgramInfo.TIdIGCompCount),
                                 false);
 
+    [[maybe_unused]] int64_t PGMRSrc3;
     assert(STM.hasGFX90AInsts() ||
-           getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx) == 0);
+           (CurrentProgramInfo.ComputePGMRSrc3GFX90A->evaluateAsAbsolute(
+                PGMRSrc3) &&
+            static_cast<uint64_t>(PGMRSrc3) == 0));
     if (STM.hasGFX90AInsts()) {
       OutStreamer->emitRawComment(
           " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
-              Twine((AMDHSA_BITS_GET(
-                  getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx),
-                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET))),
+              getMCExprStr(MCKernelDescriptor::bits_get(
+                  CurrentProgramInfo.ComputePGMRSrc3GFX90A,
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
           false);
       OutStreamer->emitRawComment(
           " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
-              Twine((AMDHSA_BITS_GET(
-                  getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx),
-                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT))),
+              getMCExprStr(MCKernelDescriptor::bits_get(
+                  CurrentProgramInfo.ComputePGMRSrc3GFX90A,
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
           false);
     }
   }
@@ -1463,28 +1490,26 @@ void AMDGPUAsmPrinter::emitResourceUsageRemarks(
   // remarks to simulate newlines. If and when clang does accept newlines, this
   // formatting should be aggregated into one remark with newlines to avoid
   // printing multiple diagnostic location and diag opts.
-  MCContext &MCCtx = MF.getContext();
   EmitResourceUsageRemark("FunctionName", "Function Name",
                           MF.getFunction().getName());
   EmitResourceUsageRemark("NumSGPR", "SGPRs",
-                          getMCExprValue(CurrentProgramInfo.NumSGPR, MCCtx));
-  EmitResourceUsageRemark(
-      "NumVGPR", "VGPRs",
-      getMCExprValue(CurrentProgramInfo.NumArchVGPR, MCCtx));
+                          getMCExprStr(CurrentProgramInfo.NumSGPR));
+  EmitResourceUsageRemark("NumVGPR", "VGPRs",
+                          getMCExprStr(CurrentProgramInfo.NumArchVGPR));
   if (hasMAIInsts) {
-    EmitResourceUsageRemark(
-        "NumAGPR", "AGPRs",
-        getMCExprValue(CurrentProgramInfo.NumAccVGPR, MCCtx));
+    EmitResourceUsageRemark("NumAGPR", "AGPRs",
+                            getMCExprStr(CurrentProgramInfo.NumAccVGPR));
   }
-  EmitResourceUsageRemark(
-      "ScratchSize", "ScratchSize [bytes/lane]",
-      getMCExprValue(CurrentProgramInfo.ScratchSize, MCCtx));
+  EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
+                          getMCExprStr(CurrentProgramInfo.ScratchSize));
+  int64_t DynStack;
+  bool DynStackEvaluatable =
+      CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(DynStack);
   StringRef DynamicStackStr =
-      getMCExprValue(CurrentProgramInfo.DynamicCallStack, MCCtx) ? "True"
-                                                                 : "False";
+      DynStackEvaluatable && DynStack ? "True" : "False";
   EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
   EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
-                          getMCExprValue(CurrentProgramInfo.Occupancy, MCCtx));
+                          getMCExprStr(CurrentProgramInfo.Occupancy));
   EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
                           CurrentProgramInfo.SGPRSpill);
   EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 87156f27fc6c5ae0fe2e64db0c47c9ed34ec5fae..162cd40687c7ee6db887b2518ece50c33ba785b8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -65,6 +65,11 @@ private:
                                   uint32_t TotalNumVGPR, uint32_t NumSGPR,
                                   uint64_t ScratchSize, uint64_t CodeSize,
                                   const AMDGPUMachineFunction *MFI);
+  void emitCommonFunctionComments(const MCExpr *NumVGPR, const MCExpr *NumAGPR,
+                                  const MCExpr *TotalNumVGPR,
+                                  const MCExpr *NumSGPR,
+                                  const MCExpr *ScratchSize, uint64_t CodeSize,
+                                  const AMDGPUMachineFunction *MFI);
   void emitResourceUsageRemarks(const MachineFunction &MF,
                                 const SIProgramInfo &CurrentProgramInfo,
                                 bool isModuleEntryFunction, bool hasMAIInsts);
@@ -79,6 +84,7 @@ private:
   void initTargetStreamer(Module &M);
 
   static uint64_t getMCExprValue(const MCExpr *Value, MCContext &Ctx);
+  SmallString<128> getMCExprStr(const MCExpr *Value);
 
 public:
   explicit AMDGPUAsmPrinter(TargetMachine &TM,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 1d645002b1fe6544db970b5e8088c85f6fda7332..38cc5a9bef969f6e6441d0751cdbd4f1f7057c5c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -249,63 +249,54 @@ void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
   switch (I.getIntrinsicID()) {
   default:
     return;
-  case Intrinsic::amdgcn_buffer_atomic_add:
   case Intrinsic::amdgcn_struct_buffer_atomic_add:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
   case Intrinsic::amdgcn_raw_buffer_atomic_add:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
     Op = AtomicRMWInst::Add;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_sub:
   case Intrinsic::amdgcn_struct_buffer_atomic_sub:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
   case Intrinsic::amdgcn_raw_buffer_atomic_sub:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
     Op = AtomicRMWInst::Sub;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_and:
   case Intrinsic::amdgcn_struct_buffer_atomic_and:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
   case Intrinsic::amdgcn_raw_buffer_atomic_and:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
     Op = AtomicRMWInst::And;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_or:
   case Intrinsic::amdgcn_struct_buffer_atomic_or:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
   case Intrinsic::amdgcn_raw_buffer_atomic_or:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
     Op = AtomicRMWInst::Or;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_xor:
   case Intrinsic::amdgcn_struct_buffer_atomic_xor:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
   case Intrinsic::amdgcn_raw_buffer_atomic_xor:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
     Op = AtomicRMWInst::Xor;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_smin:
   case Intrinsic::amdgcn_struct_buffer_atomic_smin:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
   case Intrinsic::amdgcn_raw_buffer_atomic_smin:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
     Op = AtomicRMWInst::Min;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_umin:
   case Intrinsic::amdgcn_struct_buffer_atomic_umin:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
   case Intrinsic::amdgcn_raw_buffer_atomic_umin:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
     Op = AtomicRMWInst::UMin;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_smax:
   case Intrinsic::amdgcn_struct_buffer_atomic_smax:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
   case Intrinsic::amdgcn_raw_buffer_atomic_smax:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
     Op = AtomicRMWInst::Max;
     break;
-  case Intrinsic::amdgcn_buffer_atomic_umax:
   case Intrinsic::amdgcn_struct_buffer_atomic_umax:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
   case Intrinsic::amdgcn_raw_buffer_atomic_umax:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 160a17584ca3a28294d439d63105b06e8206659c..93bca4402ed23aba1a30ad9e9730939198587e0e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1158,12 +1158,10 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       return IC.replaceInstUsesWith(II, ConstantInt::getFalse(II.getType()));
     break;
   }
-  case Intrinsic::amdgcn_buffer_store_format:
   case Intrinsic::amdgcn_raw_buffer_store_format:
   case Intrinsic::amdgcn_struct_buffer_store_format:
   case Intrinsic::amdgcn_raw_tbuffer_store:
   case Intrinsic::amdgcn_struct_tbuffer_store:
-  case Intrinsic::amdgcn_tbuffer_store:
   case Intrinsic::amdgcn_image_store_1d:
   case Intrinsic::amdgcn_image_store_1darray:
   case Intrinsic::amdgcn_image_store_2d:
@@ -1376,8 +1374,6 @@ std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
     std::function<void(Instruction *, unsigned, APInt, APInt &)>
         SimplifyAndSetOp) const {
   switch (II.getIntrinsicID()) {
-  case Intrinsic::amdgcn_buffer_load:
-  case Intrinsic::amdgcn_buffer_load_format:
   case Intrinsic::amdgcn_raw_buffer_load:
   case Intrinsic::amdgcn_raw_ptr_buffer_load:
   case Intrinsic::amdgcn_raw_buffer_load_format:
@@ -1391,7 +1387,6 @@ std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
   case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
   case Intrinsic::amdgcn_struct_tbuffer_load:
   case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
-  case Intrinsic::amdgcn_tbuffer_load:
     return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
   default: {
     if (getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID())) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 625ac0230f16066541d5d304913fd945b89c3d00..2bdbf4151dd954e41dfca867fc146813f97e6531 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -1017,7 +1017,7 @@ public:
       //
       // TODO: We could filter out subgraphs that do not access LDS globals.
       for (Function *F : KernelsThatAllocateTableLDS)
-        removeFnAttrFromReachable(CG, F, "amdgpu-no-lds-kernel-id");
+        removeFnAttrFromReachable(CG, F, {"amdgpu-no-lds-kernel-id"});
     }
 
     DenseMap<Function *, GlobalVariable *> KernelToCreatedDynamicLDS =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 410dc83d45c57fb90675b5b2cb354e070645259a..e84d39a2895c8808c4c7cb12f2a72085ea2a3899 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -256,17 +256,6 @@ def : SourceOfDivergence<int_amdgcn_ds_fadd>;
 def : SourceOfDivergence<int_amdgcn_ds_fmin>;
 def : SourceOfDivergence<int_amdgcn_ds_fmax>;
 def : SourceOfDivergence<int_amdgcn_ds_fadd_v2bf16>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_swap>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_add>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_sub>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_smin>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_umin>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_smax>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_umax>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_and>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_or>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_xor>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_cmpswap>;
 def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_swap>;
 def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_add>;
 def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_sub>;
@@ -339,7 +328,6 @@ def : SourceOfDivergence<int_amdgcn_struct_ptr_buffer_atomic_fmin>;
 def : SourceOfDivergence<int_amdgcn_struct_ptr_buffer_atomic_fmax>;
 def : SourceOfDivergence<int_amdgcn_struct_ptr_buffer_atomic_cmpswap>;
 def : SourceOfDivergence<int_amdgcn_struct_ptr_buffer_atomic_cond_sub_u32>;
-def : SourceOfDivergence<int_amdgcn_buffer_atomic_csub>;
 def : SourceOfDivergence<int_amdgcn_ps_live>;
 def : SourceOfDivergence<int_amdgcn_live_mask>;
 def : SourceOfDivergence<int_amdgcn_ds_swizzle>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 94ee4ac78142d33e0c91a0b88b1c933b52d0df84..0751c8dc8b8bf68197614eec25b55769148601ef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -560,10 +560,16 @@ bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
   else
     ++MaxSize;
 
-  MDBuilder MDB(I->getContext());
-  MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
-                                                  APInt(32, MaxSize));
-  I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
+  APInt Lower{32, MinSize};
+  APInt Upper{32, MaxSize};
+  if (auto *CI = dyn_cast<CallBase>(I)) {
+    ConstantRange Range(Lower, Upper);
+    CI->addRangeRetAttr(Range);
+  } else {
+    MDBuilder MDB(I->getContext());
+    MDNode *MaxWorkGroupSizeRange = MDB.createRange(Lower, Upper);
+    I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
+  }
   return true;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 9c94ca1e47080216048a9c6b4f4cf6c210a23967..ce997c659094a8c6662aa0c9a673900fa3313c35 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -57,6 +57,7 @@
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/IPO/ExpandVariadics.h"
 #include "llvm/Transforms/IPO/GlobalDCE.h"
 #include "llvm/Transforms/IPO/Internalize.h"
 #include "llvm/Transforms/Scalar.h"
@@ -739,6 +740,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(
         if (EnableLowerModuleLDS)
           PM.addPass(AMDGPULowerModuleLDSPass(*this));
       });
+
+  PB.registerRegClassFilterParsingCallback(
+      [](StringRef FilterName) -> RegClassFilterFunc {
+        if (FilterName == "sgpr")
+          return onlyAllocateSGPRs;
+        if (FilterName == "vgpr")
+          return onlyAllocateVGPRs;
+        return nullptr;
+      });
 }
 
 int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) {
@@ -992,6 +1002,10 @@ void AMDGPUPassConfig::addIRPasses() {
   if (isPassEnabled(EnableImageIntrinsicOptimizer))
     addPass(createAMDGPUImageIntrinsicOptimizerPass(&TM));
 
+  // This can be disabled by passing ::Disable here or on the command line
+  // with --expand-variadics-override=disable.
+  addPass(createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
+
   // Function calls are not supported, so make sure we inline everything.
   addPass(createAMDGPUAlwaysInlinePass());
   addPass(createAlwaysInlinerLegacyPass());
diff --git a/llvm/lib/Target/AMDGPU/GCNProcessors.td b/llvm/lib/Target/AMDGPU/GCNProcessors.td
index 2ada981a77cd860b6c14aad31f28fcfc6a5de2da..d218ffeb1fec769dd95fdf5ffacf06370fa04d8b 100644
--- a/llvm/lib/Target/AMDGPU/GCNProcessors.td
+++ b/llvm/lib/Target/AMDGPU/GCNProcessors.td
@@ -295,7 +295,11 @@ def : ProcessorModel<"gfx1151", GFX11SpeedModel,
   FeatureISAVersion11_5_1.Features
 >;
 
-// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151]
+def : ProcessorModel<"gfx1152", GFX11SpeedModel,
+  FeatureISAVersion11_5_2.Features
+>;
+
+// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152]
 def : ProcessorModel<"gfx11-generic", GFX11SpeedModel,
   FeatureISAVersion11_Generic.Features
 >;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index d7d6e00d2389761e8bb78bf9fb6279701e175b5e..e805e964ffe4e5860e6ab6f6e188da6c3d5f5049 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -113,6 +113,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) {
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103: AK = GK_GFX1103; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150: AK = GK_GFX1150; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: AK = GK_GFX1151; break;
+  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: AK = GK_GFX1152; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC:     AK = GK_GFX9_GENERIC; break;
@@ -196,6 +197,7 @@ unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {
   case GK_GFX1103: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103;
   case GK_GFX1150: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150;
   case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151;
+  case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152;
   case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;
   case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;
   case GK_GFX9_GENERIC:     return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4d8667affdb4d8c0a6ba7818a7fb9e5675ce6c0d..77b2af7c795851913dacebd461f5b620c77f5b0c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1280,19 +1280,6 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
     return true;
   }
-  case Intrinsic::amdgcn_buffer_atomic_fadd: {
-    Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
-    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
-    Info.align.reset();
-    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
-
-    const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
-    if (!Vol || !Vol->isZero())
-      Info.flags |= MachineMemOperand::MOVolatile;
-
-    return true;
-  }
   case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
   case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
@@ -8609,12 +8596,6 @@ SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
                                  M->getMemOperand());
 }
 
-// Return a value to use for the idxen operand by examining the vindex operand.
-static unsigned getIdxEn(SDValue VIndex) {
-  // No need to set idxen if vindex is known to be zero.
-  return isNullConstant(VIndex) ? 0 : 1;
-}
-
 SDValue
 SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
                                                 unsigned NewOpcode) const {
@@ -8739,43 +8720,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     return DAG.getMemIntrinsicNode(Opc, SDLoc(Op), M->getVTList(), Ops,
                                    M->getMemoryVT(), M->getMemOperand());
   }
-  case Intrinsic::amdgcn_buffer_load:
-  case Intrinsic::amdgcn_buffer_load_format: {
-    unsigned Glc = Op.getConstantOperandVal(5);
-    unsigned Slc = Op.getConstantOperandVal(6);
-    unsigned IdxEn = getIdxEn(Op.getOperand(3));
-    SDValue Ops[] = {
-      Op.getOperand(0), // Chain
-      Op.getOperand(2), // rsrc
-      Op.getOperand(3), // vindex
-      SDValue(),        // voffset -- will be set by setBufferOffsets
-      SDValue(),        // soffset -- will be set by setBufferOffsets
-      SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
-    };
-    setBufferOffsets(Op.getOperand(4), DAG, &Ops[3]);
-
-    unsigned Opc = (IntrID == Intrinsic::amdgcn_buffer_load) ?
-        AMDGPUISD::BUFFER_LOAD : AMDGPUISD::BUFFER_LOAD_FORMAT;
-
-    EVT VT = Op.getValueType();
-    EVT IntVT = VT.changeTypeToInteger();
-    auto *M = cast<MemSDNode>(Op);
-    EVT LoadVT = Op.getValueType();
-
-    if (LoadVT.getScalarType() == MVT::f16)
-      return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16,
-                                 M, DAG, Ops);
-
-    // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
-    if (LoadVT.getScalarType() == MVT::i8 || LoadVT.getScalarType() == MVT::i16)
-      return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops,
-                                        M->getMemOperand());
-
-    return getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops, IntVT,
-                               M->getMemOperand(), DAG);
-  }
   case Intrinsic::amdgcn_raw_buffer_load:
   case Intrinsic::amdgcn_raw_ptr_buffer_load:
   case Intrinsic::amdgcn_raw_buffer_load_format:
@@ -8825,35 +8769,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
 
     return lowerIntrinsicLoad(cast<MemSDNode>(Op), IsFormat, DAG, Ops);
   }
-  case Intrinsic::amdgcn_tbuffer_load: {
-    MemSDNode *M = cast<MemSDNode>(Op);
-    EVT LoadVT = Op.getValueType();
-
-    auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
-    unsigned Dfmt = Op.getConstantOperandVal(7);
-    unsigned Nfmt = Op.getConstantOperandVal(8);
-    unsigned Glc = Op.getConstantOperandVal(9);
-    unsigned Slc = Op.getConstantOperandVal(10);
-    unsigned IdxEn = getIdxEn(Op.getOperand(3));
-    SDValue Ops[] = {
-        Op.getOperand(0),                                        // Chain
-        Op.getOperand(2),                                        // rsrc
-        Op.getOperand(3),                                        // vindex
-        Op.getOperand(4),                                        // voffset
-        SOffset,                                                 // soffset
-        Op.getOperand(6),                                        // offset
-        DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
-        DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32),   // cachepolicy
-        DAG.getTargetConstant(IdxEn, DL, MVT::i1)                // idxen
-    };
-
-    if (LoadVT.getScalarType() == MVT::f16)
-      return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16,
-                                 M, DAG, Ops);
-    return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
-                               Op->getVTList(), Ops, LoadVT, M->getMemOperand(),
-                               DAG);
-  }
   case Intrinsic::amdgcn_raw_tbuffer_load:
   case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
     MemSDNode *M = cast<MemSDNode>(Op);
@@ -8908,82 +8823,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                Op->getVTList(), Ops, LoadVT, M->getMemOperand(),
                                DAG);
   }
-  case Intrinsic::amdgcn_buffer_atomic_swap:
-  case Intrinsic::amdgcn_buffer_atomic_add:
-  case Intrinsic::amdgcn_buffer_atomic_sub:
-  case Intrinsic::amdgcn_buffer_atomic_csub:
-  case Intrinsic::amdgcn_buffer_atomic_smin:
-  case Intrinsic::amdgcn_buffer_atomic_umin:
-  case Intrinsic::amdgcn_buffer_atomic_smax:
-  case Intrinsic::amdgcn_buffer_atomic_umax:
-  case Intrinsic::amdgcn_buffer_atomic_and:
-  case Intrinsic::amdgcn_buffer_atomic_or:
-  case Intrinsic::amdgcn_buffer_atomic_xor:
-  case Intrinsic::amdgcn_buffer_atomic_fadd: {
-    unsigned Slc = Op.getConstantOperandVal(6);
-    unsigned IdxEn = getIdxEn(Op.getOperand(4));
-    SDValue Ops[] = {
-      Op.getOperand(0), // Chain
-      Op.getOperand(2), // vdata
-      Op.getOperand(3), // rsrc
-      Op.getOperand(4), // vindex
-      SDValue(),        // voffset -- will be set by setBufferOffsets
-      SDValue(),        // soffset -- will be set by setBufferOffsets
-      SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy
-      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
-    };
-    setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
-
-    EVT VT = Op.getValueType();
-
-    auto *M = cast<MemSDNode>(Op);
-    unsigned Opcode = 0;
-
-    switch (IntrID) {
-    case Intrinsic::amdgcn_buffer_atomic_swap:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_SWAP;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_add:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_ADD;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_sub:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_SUB;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_csub:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_CSUB;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_smin:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_SMIN;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_umin:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_UMIN;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_smax:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_SMAX;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_umax:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_UMAX;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_and:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_AND;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_or:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_OR;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_xor:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_XOR;
-      break;
-    case Intrinsic::amdgcn_buffer_atomic_fadd:
-      Opcode = AMDGPUISD::BUFFER_ATOMIC_FADD;
-      break;
-    default:
-      llvm_unreachable("unhandled atomic opcode");
-    }
-
-    return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT,
-                                   M->getMemOperand());
-  }
   case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
     return lowerRawBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FADD);
@@ -9092,29 +8931,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     return lowerStructBufferAtomicIntrin(Op, DAG,
                                          AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
 
-  case Intrinsic::amdgcn_buffer_atomic_cmpswap: {
-    unsigned Slc = Op.getConstantOperandVal(7);
-    unsigned IdxEn = getIdxEn(Op.getOperand(5));
-    SDValue Ops[] = {
-      Op.getOperand(0), // Chain
-      Op.getOperand(2), // src
-      Op.getOperand(3), // cmp
-      Op.getOperand(4), // rsrc
-      Op.getOperand(5), // vindex
-      SDValue(),        // voffset -- will be set by setBufferOffsets
-      SDValue(),        // soffset -- will be set by setBufferOffsets
-      SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy
-      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
-    };
-    setBufferOffsets(Op.getOperand(6), DAG, &Ops[5]);
-
-    EVT VT = Op.getValueType();
-    auto *M = cast<MemSDNode>(Op);
-
-    return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL,
-                                   Op->getVTList(), Ops, VT, M->getMemOperand());
-  }
   case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
     SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(4), DAG);
@@ -9557,34 +9373,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
 
     return SDValue();
   };
-  case Intrinsic::amdgcn_tbuffer_store: {
-    SDValue VData = Op.getOperand(2);
-    bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
-    if (IsD16)
-      VData = handleD16VData(VData, DAG);
-    unsigned Dfmt = Op.getConstantOperandVal(8);
-    unsigned Nfmt = Op.getConstantOperandVal(9);
-    unsigned Glc = Op.getConstantOperandVal(10);
-    unsigned Slc = Op.getConstantOperandVal(11);
-    unsigned IdxEn = getIdxEn(Op.getOperand(4));
-    SDValue Ops[] = {
-      Chain,
-      VData,             // vdata
-      Op.getOperand(3),  // rsrc
-      Op.getOperand(4),  // vindex
-      Op.getOperand(5),  // voffset
-      Op.getOperand(6),  // soffset
-      Op.getOperand(7),  // offset
-      DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
-      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
-    };
-    unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 :
-                           AMDGPUISD::TBUFFER_STORE_FORMAT;
-    MemSDNode *M = cast<MemSDNode>(Op);
-    return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops,
-                                   M->getMemoryVT(), M->getMemOperand());
-  }
 
   case Intrinsic::amdgcn_struct_tbuffer_store:
   case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
@@ -9642,42 +9430,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                                    M->getMemoryVT(), M->getMemOperand());
   }
 
-  case Intrinsic::amdgcn_buffer_store:
-  case Intrinsic::amdgcn_buffer_store_format: {
-    SDValue VData = Op.getOperand(2);
-    bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
-    if (IsD16)
-      VData = handleD16VData(VData, DAG);
-    unsigned Glc = Op.getConstantOperandVal(6);
-    unsigned Slc = Op.getConstantOperandVal(7);
-    unsigned IdxEn = getIdxEn(Op.getOperand(4));
-    SDValue Ops[] = {
-      Chain,
-      VData,
-      Op.getOperand(3), // rsrc
-      Op.getOperand(4), // vindex
-      SDValue(), // voffset -- will be set by setBufferOffsets
-      SDValue(), // soffset -- will be set by setBufferOffsets
-      SDValue(), // offset -- will be set by setBufferOffsets
-      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
-    };
-    setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
-
-    unsigned Opc = IntrinsicID == Intrinsic::amdgcn_buffer_store ?
-                   AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
-    Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
-    MemSDNode *M = cast<MemSDNode>(Op);
-
-    // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
-    EVT VDataType = VData.getValueType().getScalarType();
-    if (VDataType == MVT::i8 || VDataType == MVT::i16)
-      return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
-
-    return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops,
-                                   M->getMemoryVT(), M->getMemOperand());
-  }
-
   case Intrinsic::amdgcn_raw_buffer_store:
   case Intrinsic::amdgcn_raw_ptr_buffer_store:
   case Intrinsic::amdgcn_raw_buffer_store_format:
@@ -10083,8 +9835,8 @@ std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
   return {N0, SDValue(C1, 0)};
 }
 
-// Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
-// three offsets (voffset, soffset and instoffset) into the SDValue[3] array
+// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
+// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
 // pointed to by Offsets.
 void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
                                         SelectionDAG &DAG, SDValue *Offsets,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 4c02bb1b306e5632a849ceb1560bd6ca6be03b8d..1f198a92c0fa6a3c0c5f1eee04a17eb48a36b0dd 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -253,9 +253,9 @@ public:
   bool shouldExpandVectorDynExt(SDNode *N) const;
 
 private:
-  // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
-  // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
-  // pointed to by Offsets.
+  // Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
+  // the three offsets (voffset, soffset and instoffset) into the SDValue[3]
+  // array pointed to by Offsets.
   void setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG,
                         SDValue *Offsets, Align Alignment = Align(4)) const;
 
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index c47eea20563dfb64b2d18f9e6d0668e51315220c..8b42d4a1dee7a91f36262ad2d5cca1652002328a 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -2052,9 +2052,6 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
     MemInfoMap &Visited,
     SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
 
-  if (!(MI.mayLoad() ^ MI.mayStore()))
-    return false;
-
   if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI))
     return false;
 
@@ -2065,10 +2062,6 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
   unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
                                               : AMDGPUAS::FLAT_ADDRESS;
 
-  if (MI.mayLoad() &&
-      TII->getNamedOperand(MI, AMDGPU::OpName::vdata) != nullptr)
-    return false;
-
   if (AnchorList.count(&MI))
     return false;
 
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index f178324dbbe246907a56041bf2b513dbf6dc2d4f..5dc3457b5bfae17f1787e6027c2ed0e5e0a7b94f 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -103,8 +103,6 @@ private:
 
   MachineBasicBlock *emitEndCf(MachineInstr &MI);
 
-  void lowerInitExec(MachineBasicBlock *MBB, MachineInstr &MI);
-
   void findMaskOperands(MachineInstr &MI, unsigned OpNo,
                         SmallVectorImpl<MachineOperand> &Src) const;
 
@@ -709,95 +707,6 @@ MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
   return SplitBB;
 }
 
-void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB,
-                                       MachineInstr &MI) {
-  MachineFunction &MF = *MBB->getParent();
-  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  bool IsWave32 = ST.isWave32();
-
-  if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {
-    // This should be before all vector instructions.
-    MachineInstr *InitMI = BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(),
-            TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), Exec)
-        .addImm(MI.getOperand(0).getImm());
-    if (LIS) {
-      LIS->RemoveMachineInstrFromMaps(MI);
-      LIS->InsertMachineInstrInMaps(*InitMI);
-    }
-    MI.eraseFromParent();
-    return;
-  }
-
-  // Extract the thread count from an SGPR input and set EXEC accordingly.
-  // Since BFM can't shift by 64, handle that case with CMP + CMOV.
-  //
-  // S_BFE_U32 count, input, {shift, 7}
-  // S_BFM_B64 exec, count, 0
-  // S_CMP_EQ_U32 count, 64
-  // S_CMOV_B64 exec, -1
-  Register InputReg = MI.getOperand(0).getReg();
-  MachineInstr *FirstMI = &*MBB->begin();
-  if (InputReg.isVirtual()) {
-    MachineInstr *DefInstr = MRI->getVRegDef(InputReg);
-    assert(DefInstr && DefInstr->isCopy());
-    if (DefInstr->getParent() == MBB) {
-      if (DefInstr != FirstMI) {
-        // If the `InputReg` is defined in current block, we also need to
-        // move that instruction to the beginning of the block.
-        DefInstr->removeFromParent();
-        MBB->insert(FirstMI, DefInstr);
-        if (LIS)
-          LIS->handleMove(*DefInstr);
-      } else {
-        // If first instruction is definition then move pointer after it.
-        FirstMI = &*std::next(FirstMI->getIterator());
-      }
-    }
-  }
-
-  // Insert instruction sequence at block beginning (before vector operations).
-  const DebugLoc DL = MI.getDebugLoc();
-  const unsigned WavefrontSize = ST.getWavefrontSize();
-  const unsigned Mask = (WavefrontSize << 1) - 1;
-  Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass);
-  auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg)
-                   .addReg(InputReg)
-                   .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
-  if (LV)
-    LV->recomputeForSingleDefVirtReg(InputReg);
-  auto BfmMI =
-      BuildMI(*MBB, FirstMI, DL,
-              TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec)
-          .addReg(CountReg)
-          .addImm(0);
-  auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
-                   .addReg(CountReg, RegState::Kill)
-                   .addImm(WavefrontSize);
-  if (LV)
-    LV->getVarInfo(CountReg).Kills.push_back(CmpMI);
-  auto CmovMI =
-      BuildMI(*MBB, FirstMI, DL,
-              TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
-              Exec)
-          .addImm(-1);
-
-  if (!LIS) {
-    MI.eraseFromParent();
-    return;
-  }
-
-  LIS->RemoveMachineInstrFromMaps(MI);
-  MI.eraseFromParent();
-
-  LIS->InsertMachineInstrInMaps(*BfeMI);
-  LIS->InsertMachineInstrInMaps(*BfmMI);
-  LIS->InsertMachineInstrInMaps(*CmpMI);
-  LIS->InsertMachineInstrInMaps(*CmovMI);
-
-  RecomputeRegs.insert(InputReg);
-  LIS->createAndComputeVirtRegInterval(CountReg);
-}
-
 bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
   for (auto &I : MBB.instrs()) {
     if (!I.isDebugInstr() && !I.isUnconditionalBranch())
@@ -927,18 +836,6 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
         SplitMBB = process(MI);
         Changed = true;
         break;
-
-      // FIXME: find a better place for this
-      case AMDGPU::SI_INIT_EXEC:
-      case AMDGPU::SI_INIT_EXEC_FROM_INPUT:
-        lowerInitExec(MBB, MI);
-        if (LIS)
-          LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
-        Changed = true;
-        break;
-
-      default:
-        break;
       }
 
       if (SplitMBB != MBB) {
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 09dc1c781e2f30fda490cdd5c82867c56f97f48b..913942dda19d91993fcf4af342878d3504fdf0de 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -177,6 +177,7 @@ private:
   SmallVector<MachineInstr *, 4> LowerToMovInstrs;
   SmallVector<MachineInstr *, 4> LowerToCopyInstrs;
   SmallVector<MachineInstr *, 4> KillInstrs;
+  SmallVector<MachineInstr *, 4> InitExecInstrs;
 
   void printInfo();
 
@@ -223,6 +224,8 @@ private:
   void lowerLiveMaskQueries();
   void lowerCopyInstrs();
   void lowerKillInstrs(bool IsWQM);
+  void lowerInitExec(MachineInstr &MI);
+  MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry);
 
 public:
   static char ID;
@@ -580,6 +583,9 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
                  Opcode == AMDGPU::SI_DEMOTE_I1) {
         KillInstrs.push_back(&MI);
         BBI.NeedsLowering = true;
+      } else if (Opcode == AMDGPU::SI_INIT_EXEC ||
+                 Opcode == AMDGPU::SI_INIT_EXEC_FROM_INPUT) {
+        InitExecInstrs.push_back(&MI);
       } else if (WQMOutputs) {
         // The function is in machine SSA form, which means that physical
         // VGPRs correspond to shader inputs and outputs. Inputs are
@@ -1556,6 +1562,111 @@ void SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
   }
 }
 
+void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) {
+  MachineBasicBlock *MBB = MI.getParent();
+  bool IsWave32 = ST->isWave32();
+
+  if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {
+    // This should be before all vector instructions.
+    MachineInstr *InitMI =
+        BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(),
+                TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
+                Exec)
+            .addImm(MI.getOperand(0).getImm());
+    if (LIS) {
+      LIS->RemoveMachineInstrFromMaps(MI);
+      LIS->InsertMachineInstrInMaps(*InitMI);
+    }
+    MI.eraseFromParent();
+    return;
+  }
+
+  // Extract the thread count from an SGPR input and set EXEC accordingly.
+  // Since BFM can't shift by 64, handle that case with CMP + CMOV.
+  //
+  // S_BFE_U32 count, input, {shift, 7}
+  // S_BFM_B64 exec, count, 0
+  // S_CMP_EQ_U32 count, 64
+  // S_CMOV_B64 exec, -1
+  Register InputReg = MI.getOperand(0).getReg();
+  MachineInstr *FirstMI = &*MBB->begin();
+  if (InputReg.isVirtual()) {
+    MachineInstr *DefInstr = MRI->getVRegDef(InputReg);
+    assert(DefInstr && DefInstr->isCopy());
+    if (DefInstr->getParent() == MBB) {
+      if (DefInstr != FirstMI) {
+        // If the `InputReg` is defined in current block, we also need to
+        // move that instruction to the beginning of the block.
+        DefInstr->removeFromParent();
+        MBB->insert(FirstMI, DefInstr);
+        if (LIS)
+          LIS->handleMove(*DefInstr);
+      } else {
+        // If first instruction is definition then move pointer after it.
+        FirstMI = &*std::next(FirstMI->getIterator());
+      }
+    }
+  }
+
+  // Insert instruction sequence at block beginning (before vector operations).
+  const DebugLoc DL = MI.getDebugLoc();
+  const unsigned WavefrontSize = ST->getWavefrontSize();
+  const unsigned Mask = (WavefrontSize << 1) - 1;
+  Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass);
+  auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg)
+                   .addReg(InputReg)
+                   .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
+  auto BfmMI =
+      BuildMI(*MBB, FirstMI, DL,
+              TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec)
+          .addReg(CountReg)
+          .addImm(0);
+  auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
+                   .addReg(CountReg, RegState::Kill)
+                   .addImm(WavefrontSize);
+  auto CmovMI =
+      BuildMI(*MBB, FirstMI, DL,
+              TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+              Exec)
+          .addImm(-1);
+
+  if (!LIS) {
+    MI.eraseFromParent();
+    return;
+  }
+
+  LIS->RemoveMachineInstrFromMaps(MI);
+  MI.eraseFromParent();
+
+  LIS->InsertMachineInstrInMaps(*BfeMI);
+  LIS->InsertMachineInstrInMaps(*BfmMI);
+  LIS->InsertMachineInstrInMaps(*CmpMI);
+  LIS->InsertMachineInstrInMaps(*CmovMI);
+
+  LIS->removeInterval(InputReg);
+  LIS->createAndComputeVirtRegInterval(InputReg);
+  LIS->createAndComputeVirtRegInterval(CountReg);
+}
+
+/// Lower INIT_EXEC instructions. Return a suitable insert point in \p Entry
+/// for instructions that depend on EXEC.
+MachineBasicBlock::iterator
+SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry) {
+  MachineBasicBlock::iterator InsertPt = Entry.getFirstNonPHI();
+
+  for (MachineInstr *MI : InitExecInstrs) {
+    // Try to handle undefined cases gracefully:
+    // - multiple INIT_EXEC instructions
+    // - INIT_EXEC instructions not in the entry block
+    if (MI->getParent() == &Entry)
+      InsertPt = std::next(MI->getIterator());
+
+    lowerInitExec(*MI);
+  }
+
+  return InsertPt;
+}
+
 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()
                     << " ------------- \n");
@@ -1567,6 +1678,7 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
   LowerToCopyInstrs.clear();
   LowerToMovInstrs.clear();
   KillInstrs.clear();
+  InitExecInstrs.clear();
   StateTransition.clear();
 
   ST = &MF.getSubtarget<GCNSubtarget>();
@@ -1603,16 +1715,16 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
 
   LiveMaskReg = Exec;
 
+  MachineBasicBlock &Entry = MF.front();
+  MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry);
+
   // Shader is simple does not need any state changes or any complex lowering
   if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
       LowerToMovInstrs.empty() && KillInstrs.empty()) {
     lowerLiveMaskQueries();
-    return !LiveMaskQueries.empty();
+    return !InitExecInstrs.empty() || !LiveMaskQueries.empty();
   }
 
-  MachineBasicBlock &Entry = MF.front();
-  MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
-
   // Store a copy of the original live mask when required
   if (NeedsLiveMask || (GlobalFlags & StateWQM)) {
     LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC());
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
index 239e0ee70572965b25a16e8c3ad9f39dfd87b7e9..04c6e940e6ed675a140aa215c388cbc1665790f5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
@@ -235,8 +235,9 @@ LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M) {
 }
 
 void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
-                               StringRef FnAttr) {
-  KernelRoot->removeFnAttr(FnAttr);
+                               ArrayRef<StringRef> FnAttrs) {
+  for (StringRef Attr : FnAttrs)
+    KernelRoot->removeFnAttr(Attr);
 
   SmallVector<Function *> WorkList = {CG[KernelRoot]->getFunction()};
   SmallPtrSet<Function *, 8> Visited;
@@ -261,12 +262,15 @@ void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
             Function *PotentialCallee =
                 ExternalCallRecord.second->getFunction();
             assert(PotentialCallee);
-            if (!isKernelLDS(PotentialCallee))
-              PotentialCallee->removeFnAttr(FnAttr);
+            if (!isKernelLDS(PotentialCallee)) {
+              for (StringRef Attr : FnAttrs)
+                PotentialCallee->removeFnAttr(Attr);
+            }
           }
         }
       } else {
-        Callee->removeFnAttr(FnAttr);
+        for (StringRef Attr : FnAttrs)
+          Callee->removeFnAttr(Attr);
         if (Visited.insert(Callee).second)
           WorkList.push_back(Callee);
       }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
index 4d3ad328e1310ff1196c845b52aff4f0a0365cda..e1cd4d03052b3cd8d8fc7af4e3cb8fdf5225225e 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
 #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 
@@ -54,7 +55,7 @@ LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M);
 /// Strip FnAttr attribute from any functions where we may have
 /// introduced its use.
 void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
-                               StringRef FnAttr);
+                               ArrayRef<StringRef> FnAttrs);
 
 /// Given a \p Def clobbering a load from \p Ptr according to the MSSA check
 /// if this is actually a memory update or an artificial clobber to facilitate
diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 9adf758b46c481eec56f0caffcd4469fbaf77155..c149db3144c7c22dab294577c40747722a1e8ee6 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -207,7 +207,7 @@ getReservedRegs(const MachineFunction &MF) const {
   markSuperRegs(Reserved, ARM::PC);
   markSuperRegs(Reserved, ARM::FPSCR);
   markSuperRegs(Reserved, ARM::APSR_NZCV);
-  if (TFI->hasFP(MF))
+  if (TFI->isFPReserved(MF))
     markSuperRegs(Reserved, STI.getFramePointerReg());
   if (hasBasePointer(MF))
     markSuperRegs(Reserved, BasePtr);
diff --git a/llvm/lib/Target/ARM/ARMFeatures.td b/llvm/lib/Target/ARM/ARMFeatures.td
index 84481af650be7a64b430da15f90b3f07322e4941..8b0ade54b46d3c1482af77ca0613f9ebf2a3fea3 100644
--- a/llvm/lib/Target/ARM/ARMFeatures.td
+++ b/llvm/lib/Target/ARM/ARMFeatures.td
@@ -548,16 +548,15 @@ def FeatureFixCortexA57AES1742098 : SubtargetFeature<"fix-cortex-a57-aes-1742098
   "FixCortexA57AES1742098", "true",
   "Work around Cortex-A57 Erratum 1742098 / Cortex-A72 Erratum 1655431 (AES)">;
 
+// If frame pointers are in use, they must follow the AAPCS definition, which
+// always uses R11 as the frame pointer. If this is not set, we can use R7 as
+// the frame pointer for Thumb1-only code, which is more efficient, but less
+// compatible. Note that this feature does not control whether frame pointers
+// are emitted, that is controlled by the "frame-pointer" function attribute.
 def FeatureAAPCSFrameChain : SubtargetFeature<"aapcs-frame-chain",
                                               "CreateAAPCSFrameChain", "true",
                                               "Create an AAPCS compliant frame chain">;
 
-def FeatureAAPCSFrameChainLeaf : SubtargetFeature<"aapcs-frame-chain-leaf",
-                                                  "CreateAAPCSFrameChainLeaf", "true",
-                                                  "Create an AAPCS compliant frame chain "
-                                                  "for leaf functions",
-                                                  [FeatureAAPCSFrameChain]>;
-
 // Assume that lock-free 32-bit atomics are available, even if the target
 // and operating system combination would not usually provide them. The user
 // is responsible for providing any necessary __sync implementations. Code
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 11496a6e032ddeab6737176a31959808c5f133c4..831b6b0fc722314547f3747f72efed77b9e92299 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -215,7 +215,7 @@ bool ARMFrameLowering::hasFP(const MachineFunction &MF) const {
 /// isFPReserved - Return true if the frame pointer register should be
 /// considered a reserved register on the scope of the specified function.
 bool ARMFrameLowering::isFPReserved(const MachineFunction &MF) const {
-  return hasFP(MF) || MF.getSubtarget<ARMSubtarget>().createAAPCSFrameChain();
+  return hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF);
 }
 
 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
@@ -2233,10 +2233,10 @@ bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
   return true;
 }
 
-static bool requiresAAPCSFrameRecord(const MachineFunction &MF) {
+bool ARMFrameLowering::requiresAAPCSFrameRecord(
+    const MachineFunction &MF) const {
   const auto &Subtarget = MF.getSubtarget<ARMSubtarget>();
-  return Subtarget.createAAPCSFrameChainLeaf() ||
-         (Subtarget.createAAPCSFrameChain() && MF.getFrameInfo().hasCalls());
+  return Subtarget.createAAPCSFrameChain() && hasFP(MF);
 }
 
 // Thumb1 may require a spill when storing to a frame index through FP (or any
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.h b/llvm/lib/Target/ARM/ARMFrameLowering.h
index 3c7358d8cd53e2a6eb6f77b16e3189f3924a8c71..6a31b73957f13485b20c32b4a879a1fb65bfc8fa 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.h
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.h
@@ -47,6 +47,7 @@ public:
 
   bool hasFP(const MachineFunction &MF) const override;
   bool isFPReserved(const MachineFunction &MF) const;
+  bool requiresAAPCSFrameRecord(const MachineFunction &MF) const;
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
   bool canSimplifyCallFramePseudos(const MachineFunction &MF) const override;
   StackOffset getFrameIndexReference(const MachineFunction &MF, int FI,
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 5212d2c620b75b06c7b934af58a81dfc0ceb83f3..e3270471981cc798291debf5fc883ca915726d35 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -205,9 +205,9 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
   setOperationAction(ISD::SDIVREM, VT, Expand);
   setOperationAction(ISD::UDIVREM, VT, Expand);
 
-  if (!VT.isFloatingPoint() &&
-      VT != MVT::v2i64 && VT != MVT::v1i64)
-    for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
+  if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
+    for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
+                        ISD::UMIN, ISD::UMAX})
       setOperationAction(Opcode, VT, Legal);
   if (!VT.isFloatingPoint())
     for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
@@ -1623,9 +1623,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setPrefFunctionAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
 
   setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
-
-  if (Subtarget->isThumb() || Subtarget->isThumb2())
-    setTargetDAGCombine(ISD::ABS);
 }
 
 bool ARMTargetLowering::useSoftFloat() const {
@@ -4174,7 +4171,15 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
   }
   case Intrinsic::arm_neon_vabs:
     return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
-                        Op.getOperand(1));
+                       Op.getOperand(1));
+  case Intrinsic::arm_neon_vabds:
+    if (Op.getValueType().isInteger())
+      return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(),
+                         Op.getOperand(1), Op.getOperand(2));
+    return SDValue();
+  case Intrinsic::arm_neon_vabdu:
+    return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::arm_neon_vmulls:
   case Intrinsic::arm_neon_vmullu: {
     unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
@@ -13496,18 +13501,6 @@ static SDValue PerformVSetCCToVCTPCombine(SDNode *N,
                          DCI.DAG.getZExtOrTrunc(Op1S, DL, MVT::i32));
 }
 
-static SDValue PerformABSCombine(SDNode *N,
-                                 TargetLowering::DAGCombinerInfo &DCI,
-                                 const ARMSubtarget *Subtarget) {
-  SelectionDAG &DAG = DCI.DAG;
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-
-  if (TLI.isOperationLegal(N->getOpcode(), N->getValueType(0)))
-    return SDValue();
-
-  return TLI.expandABS(N, DAG);
-}
-
 /// PerformADDECombine - Target-specific dag combine transform from
 /// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
 /// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
@@ -18871,7 +18864,6 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::SELECT:     return PerformSELECTCombine(N, DCI, Subtarget);
   case ISD::VSELECT:    return PerformVSELECTCombine(N, DCI, Subtarget);
   case ISD::SETCC:      return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
-  case ISD::ABS:        return PerformABSCombine(N, DCI, Subtarget);
   case ARMISD::ADDE:    return PerformADDECombine(N, DCI, Subtarget);
   case ARMISD::UMLAL:   return PerformUMLALCombine(N, DCI.DAG, Subtarget);
   case ISD::ADD:        return PerformADDCombine(N, DCI, Subtarget);
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index 21a5817252aeaac016165b783db0474deaa4fc97..fcabc9076e4d3083918001c12d87fd32e8db351f 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -5640,10 +5640,10 @@ def  VBITq    : N3VX<1, 0, 0b10, 0b0001, 1, 1,
 //   VABD     : Vector Absolute Difference
 defm VABDs    : N3VInt_QHS<0, 0, 0b0111, 0, N3RegFrm,
                            IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q,
-                           "vabd", "s", int_arm_neon_vabds, 1>;
+                           "vabd", "s", abds, 1>;
 defm VABDu    : N3VInt_QHS<1, 0, 0b0111, 0, N3RegFrm,
                            IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q,
-                           "vabd", "u", int_arm_neon_vabdu, 1>;
+                           "vabd", "u", abdu, 1>;
 def  VABDfd   : N3VDInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBIND,
                         "vabd", "f32", v2f32, v2f32, int_arm_neon_vabds, 1>;
 def  VABDfq   : N3VQInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBINQ,
@@ -5657,44 +5657,30 @@ def  VABDhq   : N3VQInt<1, 0, 0b11, 0b1101, 0, N3RegFrm, IIC_VBINQ,
 
 //   VABDL    : Vector Absolute Difference Long (Q = | D - D |)
 defm VABDLs   : N3VLIntExt_QHS<0,1,0b0111,0, IIC_VSUBi4Q,
-                               "vabdl", "s", int_arm_neon_vabds, zext, 1>;
+                               "vabdl", "s", abds, zext, 1>;
 defm VABDLu   : N3VLIntExt_QHS<1,1,0b0111,0, IIC_VSUBi4Q,
-                               "vabdl", "u", int_arm_neon_vabdu, zext, 1>;
+                               "vabdl", "u", abdu, zext, 1>;
 
 let Predicates = [HasNEON] in {
-def : Pat<(v8i16 (abs (sub (zext (v8i8 DPR:$opA)), (zext (v8i8 DPR:$opB))))),
+def : Pat<(v8i16 (zext (abdu (v8i8 DPR:$opA), (v8i8 DPR:$opB)))),
           (VABDLuv8i16 DPR:$opA, DPR:$opB)>;
-def : Pat<(v4i32 (abs (sub (zext (v4i16 DPR:$opA)), (zext (v4i16 DPR:$opB))))),
+def : Pat<(v4i32 (zext (abdu (v4i16 DPR:$opA), (v4i16 DPR:$opB)))),
           (VABDLuv4i32 DPR:$opA, DPR:$opB)>;
-}
-
-// ISD::ABS is not legal for v2i64, so VABDL needs to be matched from the
-// shift/xor pattern for ABS.
-
-def abd_shr :
-    PatFrag<(ops node:$in1, node:$in2, node:$shift),
-            (ARMvshrsImm (sub (zext node:$in1),
-                            (zext node:$in2)), (i32 $shift))>;
-
-let Predicates = [HasNEON] in {
-def : Pat<(xor (v2i64 (abd_shr (v2i32 DPR:$opA), (v2i32 DPR:$opB), 63)),
-               (v2i64 (add (sub (zext (v2i32 DPR:$opA)),
-                                (zext (v2i32 DPR:$opB))),
-                           (abd_shr (v2i32 DPR:$opA), (v2i32 DPR:$opB), 63)))),
+def : Pat<(v2i64 (zext (abdu (v2i32 DPR:$opA), (v2i32 DPR:$opB)))),
           (VABDLuv2i64 DPR:$opA, DPR:$opB)>;
 }
 
 //   VABA     : Vector Absolute Difference and Accumulate
 defm VABAs    : N3VIntOp_QHS<0,0,0b0111,1, IIC_VABAD, IIC_VABAQ,
-                             "vaba", "s", int_arm_neon_vabds, add>;
+                             "vaba", "s", abds, add>;
 defm VABAu    : N3VIntOp_QHS<1,0,0b0111,1, IIC_VABAD, IIC_VABAQ,
-                             "vaba", "u", int_arm_neon_vabdu, add>;
+                             "vaba", "u", abdu, add>;
 
 //   VABAL    : Vector Absolute Difference and Accumulate Long (Q += | D - D |)
 defm VABALs   : N3VLIntExtOp_QHS<0,1,0b0101,0, IIC_VABAD,
-                                 "vabal", "s", int_arm_neon_vabds, zext, add>;
+                                 "vabal", "s", abds, zext, add>;
 defm VABALu   : N3VLIntExtOp_QHS<1,1,0b0101,0, IIC_VABAD,
-                                 "vabal", "u", int_arm_neon_vabdu, zext, add>;
+                                 "vabal", "u", abdu, zext, add>;
 
 // Vector Maximum and Minimum.
 
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index a46c383115e2d6196dd03c4d7c2ccfe284f94172..919828753f45a3e5100eb6ae52138810e204bdbe 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -115,6 +115,12 @@ static bool shouldInspect(MachineInstr &MI) {
   return isDomainMVE(&MI) || isVectorPredicate(&MI) || hasVPRUse(MI);
 }
 
+static bool isHorizontalReduction(const MachineInstr &MI) {
+  const MCInstrDesc &MCID = MI.getDesc();
+  uint64_t Flags = MCID.TSFlags;
+  return (Flags & ARMII::HorizontalReduction) != 0;
+}
+
 namespace {
 
   using InstSet = SmallPtrSetImpl<MachineInstr *>;
@@ -275,6 +281,16 @@ namespace {
       if (VPT->getOpcode() == ARM::MVE_VPST)
         return false;
 
+      // If the VPT block does not define something that is an "output", then
+      // the tail-predicated version will just perform a subset of the original
+      // vpt block, where the last lanes should not be used.
+      if (isVPTOpcode(VPT->getOpcode()) &&
+          all_of(Block.getInsts(), [](const MachineInstr *MI) {
+            return !MI->mayStore() && !MI->mayLoad() &&
+                   !isHorizontalReduction(*MI) && !isVCTP(MI);
+          }))
+        return true;
+
       auto IsOperandPredicated = [&](MachineInstr *MI, unsigned Idx) {
         MachineInstr *Op = RDA.getMIOperand(MI, MI->getOperand(Idx));
         return Op && PredicatedInsts.count(Op) && isPredicatedOnVCTP(Op);
@@ -813,12 +829,6 @@ static bool producesDoubleWidthResult(const MachineInstr &MI) {
   return (Flags & ARMII::DoubleWidthResult) != 0;
 }
 
-static bool isHorizontalReduction(const MachineInstr &MI) {
-  const MCInstrDesc &MCID = MI.getDesc();
-  uint64_t Flags = MCID.TSFlags;
-  return (Flags & ARMII::HorizontalReduction) != 0;
-}
-
 // Can this instruction generate a non-zero result when given only zeroed
 // operands? This allows us to know that, given operands with false bytes
 // zeroed by masked loads, that the result will also contain zeros in those
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index eb5ed41ae8a17c5c2099476a12297e9f28e39628..e4e122a0d1339ba586f664d8223094ffb902dd91 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -90,6 +90,8 @@ def ProcR7      : SubtargetFeature<"r7", "ARMProcFamily", "CortexR7",
                                    "Cortex-R7 ARM processors", []>;
 def ProcR52     : SubtargetFeature<"r52", "ARMProcFamily", "CortexR52",
                                    "Cortex-R52 ARM processors", []>;
+def ProcR52plus  : SubtargetFeature<"r52plus", "ARMProcFamily", "CortexR52plus",
+                                   "Cortex-R52plus ARM processors", []>;
 
 def ProcM3      : SubtargetFeature<"m3", "ARMProcFamily", "CortexM3",
                                    "Cortex-M3 ARM processors", []>;
@@ -577,3 +579,9 @@ def : ProcessorModel<"cortex-r52", CortexR52Model,      [ARMv8r, ProcR52,
                                                          FeatureNEON,
                                                          FeatureUseMISched,
                                                          FeatureFPAO]>;
+
+def : ProcessorModel<"cortex-r52plus", CortexR52Model,  [ARMv8r, ProcR52plus,
+                                                         FeatureFPARMv8,
+                                                         FeatureNEON,
+                                                         FeatureUseMISched,
+                                                         FeatureFPAO]>;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 5e13d8fabe0485dbb7af8ebc9a5038f629ac9c70..a8c6cd99633f3ed36c5c261827cacd5e65310880 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -298,6 +298,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   case CortexM3:
   case CortexM7:
   case CortexR52:
+  case CortexR52plus:
   case CortexX1:
   case CortexX1C:
     break;
diff --git a/llvm/lib/Target/LoongArch/CMakeLists.txt b/llvm/lib/Target/LoongArch/CMakeLists.txt
index 5085e23f82a7b62c91e6f583088c89fe02026f48..cadc335a621f2e5489bc2a1572e6be79258bd029 100644
--- a/llvm/lib/Target/LoongArch/CMakeLists.txt
+++ b/llvm/lib/Target/LoongArch/CMakeLists.txt
@@ -16,6 +16,7 @@ add_public_tablegen_target(LoongArchCommonTableGen)
 
 add_llvm_target(LoongArchCodeGen
   LoongArchAsmPrinter.cpp
+  LoongArchDeadRegisterDefinitions.cpp
   LoongArchExpandAtomicPseudoInsts.cpp
   LoongArchExpandPseudoInsts.cpp
   LoongArchFrameLowering.cpp
diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h
index 0928ea31054a4b857653989a2e45aa9d8e036b09..adfb844ee31b649a9b4a3365b44c8694887b0f78 100644
--- a/llvm/lib/Target/LoongArch/LoongArch.h
+++ b/llvm/lib/Target/LoongArch/LoongArch.h
@@ -33,12 +33,14 @@ bool lowerLoongArchMachineOperandToMCOperand(const MachineOperand &MO,
                                              MCOperand &MCOp,
                                              const AsmPrinter &AP);
 
+FunctionPass *createLoongArchDeadRegisterDefinitionsPass();
 FunctionPass *createLoongArchExpandAtomicPseudoPass();
 FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM);
 FunctionPass *createLoongArchOptWInstrsPass();
 FunctionPass *createLoongArchPreRAExpandPseudoPass();
 FunctionPass *createLoongArchExpandPseudoPass();
 void initializeLoongArchDAGToDAGISelLegacyPass(PassRegistry &);
+void initializeLoongArchDeadRegisterDefinitionsPass(PassRegistry &);
 void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &);
 void initializeLoongArchOptWInstrsPass(PassRegistry &);
 void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &);
diff --git a/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6ff9b6ac25b7c3cc1ef843cf49f0fd4b4f8afbe4
--- /dev/null
+++ b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp
@@ -0,0 +1,108 @@
+//=== LoongArchDeadRegisterDefinitions.cpp - Replace dead defs w/ zero reg ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This pass rewrites Rd to r0 for instrs whose return values are unused.
+//
+//===---------------------------------------------------------------------===//
+
+#include "LoongArch.h"
+#include "LoongArchInstrInfo.h"
+#include "LoongArchSubtarget.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveStacks.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+#define DEBUG_TYPE "loongarch-dead-defs"
+#define LoongArch_DEAD_REG_DEF_NAME "LoongArch Dead register definitions"
+
+STATISTIC(NumDeadDefsReplaced, "Number of dead definitions replaced");
+
+namespace {
+class LoongArchDeadRegisterDefinitions : public MachineFunctionPass {
+public:
+  static char ID;
+
+  LoongArchDeadRegisterDefinitions() : MachineFunctionPass(ID) {}
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<LiveIntervals>();
+    AU.addPreserved<LiveIntervals>();
+    AU.addRequired<LiveIntervals>();
+    AU.addPreserved<SlotIndexes>();
+    AU.addPreserved<LiveDebugVariables>();
+    AU.addPreserved<LiveStacks>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return LoongArch_DEAD_REG_DEF_NAME; }
+};
+} // end anonymous namespace
+
+char LoongArchDeadRegisterDefinitions::ID = 0;
+INITIALIZE_PASS(LoongArchDeadRegisterDefinitions, DEBUG_TYPE,
+                LoongArch_DEAD_REG_DEF_NAME, false, false)
+
+FunctionPass *llvm::createLoongArchDeadRegisterDefinitionsPass() {
+  return new LoongArchDeadRegisterDefinitions();
+}
+
+bool LoongArchDeadRegisterDefinitions::runOnMachineFunction(
+    MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  LiveIntervals &LIS = getAnalysis<LiveIntervals>();
+  LLVM_DEBUG(dbgs() << "***** LoongArchDeadRegisterDefinitions *****\n");
+
+  bool MadeChange = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      // We only handle non-computational instructions.
+      const MCInstrDesc &Desc = MI.getDesc();
+      if (!Desc.mayLoad() && !Desc.mayStore() &&
+          !Desc.hasUnmodeledSideEffects())
+        continue;
+      for (int I = 0, E = Desc.getNumDefs(); I != E; ++I) {
+        MachineOperand &MO = MI.getOperand(I);
+        if (!MO.isReg() || !MO.isDef() || MO.isEarlyClobber())
+          continue;
+        // Be careful not to change the register if it's a tied operand.
+        if (MI.isRegTiedToUseOperand(I)) {
+          LLVM_DEBUG(dbgs() << "    Ignoring, def is tied operand.\n");
+          continue;
+        }
+        Register Reg = MO.getReg();
+        if (!Reg.isVirtual() || !MO.isDead())
+          continue;
+        LLVM_DEBUG(dbgs() << "    Dead def operand #" << I << " in:\n      ";
+                   MI.print(dbgs()));
+        const TargetRegisterClass *RC = TII->getRegClass(Desc, I, TRI, MF);
+        if (!(RC && RC->contains(LoongArch::R0))) {
+          LLVM_DEBUG(dbgs() << "    Ignoring, register is not a GPR.\n");
+          continue;
+        }
+        assert(LIS.hasInterval(Reg));
+        LIS.removeInterval(Reg);
+        MO.setReg(LoongArch::R0);
+        LLVM_DEBUG(dbgs() << "    Replacing with zero register. New:\n      ";
+                   MI.print(dbgs()));
+        ++NumDeadDefsReplaced;
+        MadeChange = true;
+      }
+    }
+  }
+
+  return MadeChange;
+}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 51384f25d24551090d4c83c24d64f0f51b05bff7..9d7e4636abac1afcf2cd2b797e67218b8a9bde9d 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -171,6 +171,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   // Set operations for 'F' feature.
 
   if (Subtarget.hasBasicF()) {
+    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
+    setTruncStoreAction(MVT::f32, MVT::f16, Expand);
     setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
 
     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
@@ -186,6 +188,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
     setOperationAction(ISD::FPOW, MVT::f32, Expand);
     setOperationAction(ISD::FREM, MVT::f32, Expand);
+    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
+    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
 
     if (Subtarget.is64Bit())
       setOperationAction(ISD::FRINT, MVT::f32, Legal);
@@ -202,7 +206,9 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   // Set operations for 'D' feature.
 
   if (Subtarget.hasBasicD()) {
+    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
+    setTruncStoreAction(MVT::f64, MVT::f16, Expand);
     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
     setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
 
@@ -219,6 +225,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
     setOperationAction(ISD::FPOW, MVT::f64, Expand);
     setOperationAction(ISD::FREM, MVT::f64, Expand);
+    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
+    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
 
     if (Subtarget.is64Bit())
       setOperationAction(ISD::FRINT, MVT::f64, Legal);
@@ -5004,6 +5012,10 @@ bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT,
   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
 }
 
+bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const {
+  return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
+}
+
 bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
   // TODO: Support vectors.
   if (Y.getValueType().isVector())
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index f274b1971fd232f62990eddac965116a42c71c07..9328831a17a30d5d708f050b35a90bb80b104330 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -229,6 +229,7 @@ public:
   bool isLegalAddImmediate(int64_t Imm) const override;
   bool isZExtFree(SDValue Val, EVT VT2) const override;
   bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override;
+  bool signExtendConstant(const ConstantInt *CI) const override;
 
   bool hasAndNotCompare(SDValue Y) const override;
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 66bd74e068b95dbf2c365e0d44e0a7953982f5c2..f72f46e39e2a4600749cd63796808521dc5e719b 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -734,7 +734,9 @@ def ADD_W : ALU_3R<0x00100000>;
 def SUB_W : ALU_3R<0x00110000>;
 def ADDI_W : ALU_2RI12<0x02800000, simm12_addlike>;
 def ALSL_W : ALU_3RI2<0x00040000, uimm2_plus1>;
+let isReMaterializable = 1 in {
 def LU12I_W : ALU_1RI20<0x14000000, simm20_lu12iw>;
+}
 def SLT  : ALU_3R<0x00120000>;
 def SLTU : ALU_3R<0x00128000>;
 def SLTI  : ALU_2RI12<0x02000000, simm12>;
@@ -749,8 +751,10 @@ def XOR  : ALU_3R<0x00158000>;
 def ANDN : ALU_3R<0x00168000>;
 def ORN  : ALU_3R<0x00160000>;
 def ANDI : ALU_2RI12<0x03400000, uimm12>;
+let isReMaterializable = 1 in {
 def ORI  : ALU_2RI12<0x03800000, uimm12_ori>;
 def XORI : ALU_2RI12<0x03c00000, uimm12>;
+}
 def MUL_W   : ALU_3R<0x001c0000>;
 def MULH_W  : ALU_3R<0x001c8000>;
 def MULH_WU : ALU_3R<0x001d0000>;
@@ -852,17 +856,24 @@ let Predicates = [IsLA64] in {
 // Arithmetic Operation Instructions for 64-bits
 def ADD_D : ALU_3R<0x00108000>;
 def SUB_D : ALU_3R<0x00118000>;
+// ADDI_D isn't always rematerializable, but isReMaterializable will be used as
+// a hint which is verified in isReallyTriviallyReMaterializable.
+let isReMaterializable = 1 in {
 def ADDI_D : ALU_2RI12<0x02c00000, simm12_addlike>;
+}
 def ADDU16I_D : ALU_2RI16<0x10000000, simm16>;
 def ALSL_WU : ALU_3RI2<0x00060000, uimm2_plus1>;
 def ALSL_D  : ALU_3RI2<0x002c0000, uimm2_plus1>;
 let Constraints = "$rd = $dst" in {
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0,
+    isReMaterializable = 1 in
 def LU32I_D : Fmt1RI20<0x16000000, (outs GPR:$dst),
                        (ins GPR:$rd, simm20_lu32id:$imm20),
                        "$rd, $imm20">;
 }
+let isReMaterializable = 1 in {
 def LU52I_D : ALU_2RI12<0x03000000, simm12_lu52id>;
+}
 def PCADDU18I : ALU_1RI20<0x1e000000, simm20_pcaddu18i>;
 def MUL_D     : ALU_3R<0x001d8000>;
 def MULH_D    : ALU_3R<0x001e0000>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
index 83466d53f84ddf5ec20de2e2e70a4753f1f39e24..137fe1d04f45b905efa69fdb33662fa02fc677ec 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
@@ -34,11 +34,19 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeLoongArchTarget() {
   RegisterTargetMachine<LoongArchTargetMachine> X(getTheLoongArch32Target());
   RegisterTargetMachine<LoongArchTargetMachine> Y(getTheLoongArch64Target());
   auto *PR = PassRegistry::getPassRegistry();
+  initializeLoongArchDeadRegisterDefinitionsPass(*PR);
   initializeLoongArchOptWInstrsPass(*PR);
   initializeLoongArchPreRAExpandPseudoPass(*PR);
   initializeLoongArchDAGToDAGISelLegacyPass(*PR);
 }
 
+static cl::opt<bool> EnableLoongArchDeadRegisterElimination(
+    "loongarch-enable-dead-defs", cl::Hidden,
+    cl::desc("Enable the pass that removes dead"
+             " definitons and replaces stores to"
+             " them with stores to r0"),
+    cl::init(true));
+
 static cl::opt<bool>
     EnableLoopDataPrefetch("loongarch-enable-loop-data-prefetch", cl::Hidden,
                            cl::desc("Enable the loop data prefetch pass"),
@@ -46,7 +54,7 @@ static cl::opt<bool>
 
 static std::string computeDataLayout(const Triple &TT) {
   if (TT.isArch64Bit())
-    return "e-m:e-p:64:64-i64:64-i128:128-n64-S128";
+    return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
   assert(TT.isArch32Bit() && "only LA32 and LA64 are currently supported");
   return "e-m:e-p:32:32-i64:64-n32-S128";
 }
@@ -148,6 +156,8 @@ public:
   void addPreEmitPass2() override;
   void addMachineSSAOptimization() override;
   void addPreRegAlloc() override;
+  bool addRegAssignAndRewriteFast() override;
+  bool addRegAssignAndRewriteOptimized() override;
 };
 } // end namespace
 
@@ -200,3 +210,17 @@ void LoongArchPassConfig::addMachineSSAOptimization() {
 void LoongArchPassConfig::addPreRegAlloc() {
   addPass(createLoongArchPreRAExpandPseudoPass());
 }
+
+bool LoongArchPassConfig::addRegAssignAndRewriteFast() {
+  if (TM->getOptLevel() != CodeGenOptLevel::None &&
+      EnableLoongArchDeadRegisterElimination)
+    addPass(createLoongArchDeadRegisterDefinitionsPass());
+  return TargetPassConfig::addRegAssignAndRewriteFast();
+}
+
+bool LoongArchPassConfig::addRegAssignAndRewriteOptimized() {
+  if (TM->getOptLevel() != CodeGenOptLevel::None &&
+      EnableLoongArchDeadRegisterElimination)
+    addPass(createLoongArchDeadRegisterDefinitionsPass());
+  return TargetPassConfig::addRegAssignAndRewriteOptimized();
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 5eefab59a6ab51e035c6478eea127d386df63413..b0cb24c63c3cebd04f4a6847307a5519180493fa 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -40,7 +40,7 @@ FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM,
 ModulePass *createNVPTXAssignValidGlobalNamesPass();
 ModulePass *createGenericToNVVMLegacyPass();
 ModulePass *createNVPTXCtorDtorLoweringLegacyPass();
-FunctionPass *createNVVMIntrRangePass(unsigned int SmVersion);
+FunctionPass *createNVVMIntrRangePass();
 FunctionPass *createNVVMReflectPass(unsigned int SmVersion);
 MachineFunctionPass *createNVPTXPrologEpilogPass();
 MachineFunctionPass *createNVPTXReplaceImageHandlesPass();
@@ -53,12 +53,7 @@ MachineFunctionPass *createNVPTXPeephole();
 MachineFunctionPass *createNVPTXProxyRegErasurePass();
 
 struct NVVMIntrRangePass : PassInfoMixin<NVVMIntrRangePass> {
-  NVVMIntrRangePass();
-  NVVMIntrRangePass(unsigned SmVersion) : SmVersion(SmVersion) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-
-private:
-  unsigned SmVersion;
 };
 
 struct NVVMReflectPass : PassInfoMixin<NVVMReflectPass> {
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index f63697916d902a1fd708d462d9f31060613d97f6..82770f8660850c04b9c7d04bc4a86007c1acf9bc 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -542,30 +542,24 @@ void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
   // If the NVVM IR has some of reqntid* specified, then output
   // the reqntid directive, and set the unspecified ones to 1.
   // If none of Reqntid* is specified, don't output reqntid directive.
-  unsigned Reqntidx, Reqntidy, Reqntidz;
-  Reqntidx = Reqntidy = Reqntidz = 1;
-  bool ReqSpecified = false;
-  ReqSpecified |= getReqNTIDx(F, Reqntidx);
-  ReqSpecified |= getReqNTIDy(F, Reqntidy);
-  ReqSpecified |= getReqNTIDz(F, Reqntidz);
+  std::optional<unsigned> Reqntidx = getReqNTIDx(F);
+  std::optional<unsigned> Reqntidy = getReqNTIDy(F);
+  std::optional<unsigned> Reqntidz = getReqNTIDz(F);
 
-  if (ReqSpecified)
-    O << ".reqntid " << Reqntidx << ", " << Reqntidy << ", " << Reqntidz
-      << "\n";
+  if (Reqntidx || Reqntidy || Reqntidz)
+    O << ".reqntid " << Reqntidx.value_or(1) << ", " << Reqntidy.value_or(1)
+      << ", " << Reqntidz.value_or(1) << "\n";
 
   // If the NVVM IR has some of maxntid* specified, then output
   // the maxntid directive, and set the unspecified ones to 1.
   // If none of maxntid* is specified, don't output maxntid directive.
-  unsigned Maxntidx, Maxntidy, Maxntidz;
-  Maxntidx = Maxntidy = Maxntidz = 1;
-  bool MaxSpecified = false;
-  MaxSpecified |= getMaxNTIDx(F, Maxntidx);
-  MaxSpecified |= getMaxNTIDy(F, Maxntidy);
-  MaxSpecified |= getMaxNTIDz(F, Maxntidz);
-
-  if (MaxSpecified)
-    O << ".maxntid " << Maxntidx << ", " << Maxntidy << ", " << Maxntidz
-      << "\n";
+  std::optional<unsigned> Maxntidx = getMaxNTIDx(F);
+  std::optional<unsigned> Maxntidy = getMaxNTIDy(F);
+  std::optional<unsigned> Maxntidz = getMaxNTIDz(F);
+
+  if (Maxntidx || Maxntidy || Maxntidz)
+    O << ".maxntid " << Maxntidx.value_or(1) << ", " << Maxntidy.value_or(1)
+      << ", " << Maxntidz.value_or(1) << "\n";
 
   unsigned Mincta = 0;
   if (getMinCTASm(F, Mincta))
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 4dc3cea4bd8e72069aae6ccdeb6cd9658c430ce3..b60a1d747af791c4ebbe958714d83c834cc48334 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -233,9 +233,9 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(
       [this](ModulePassManager &PM, OptimizationLevel Level) {
         FunctionPassManager FPM;
         FPM.addPass(NVVMReflectPass(Subtarget.getSmVersion()));
-        // FIXME: NVVMIntrRangePass is causing numerical discrepancies,
-        // investigate and re-enable.
-        // FPM.addPass(NVVMIntrRangePass(Subtarget.getSmVersion()));
+        // Note: NVVMIntrRangePass was causing numerical discrepancies at one
+        // point, if issues crop up, consider disabling.
+        FPM.addPass(NVVMIntrRangePass());
         PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
       });
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 013afe916e86c8b39c4690591525e201dceae5e7..3a536db1c97279a4b58aeef07d0ba737da717c67 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -128,6 +128,14 @@ bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
   return true;
 }
 
+static std::optional<unsigned>
+findOneNVVMAnnotation(const GlobalValue &GV, const std::string &PropName) {
+  unsigned RetVal;
+  if (findOneNVVMAnnotation(&GV, PropName, RetVal))
+    return RetVal;
+  return std::nullopt;
+}
+
 bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
                            std::vector<unsigned> &retval) {
   auto &AC = getAnnotationCache();
@@ -252,32 +260,57 @@ std::string getSamplerName(const Value &val) {
   return std::string(val.getName());
 }
 
-bool getMaxNTIDx(const Function &F, unsigned &x) {
-  return findOneNVVMAnnotation(&F, "maxntidx", x);
+std::optional<unsigned> getMaxNTIDx(const Function &F) {
+  return findOneNVVMAnnotation(F, "maxntidx");
 }
 
-bool getMaxNTIDy(const Function &F, unsigned &y) {
-  return findOneNVVMAnnotation(&F, "maxntidy", y);
+std::optional<unsigned> getMaxNTIDy(const Function &F) {
+  return findOneNVVMAnnotation(F, "maxntidy");
 }
 
-bool getMaxNTIDz(const Function &F, unsigned &z) {
-  return findOneNVVMAnnotation(&F, "maxntidz", z);
+std::optional<unsigned> getMaxNTIDz(const Function &F) {
+  return findOneNVVMAnnotation(F, "maxntidz");
+}
+
+std::optional<unsigned> getMaxNTID(const Function &F) {
+  // Note: The semantics here are a bit strange. The PTX ISA states the
+  // following (11.4.2. Performance-Tuning Directives: .maxntid):
+  //
+  //  Note that this directive guarantees that the total number of threads does
+  //  not exceed the maximum, but does not guarantee that the limit in any
+  //  particular dimension is not exceeded.
+  std::optional<unsigned> MaxNTIDx = getMaxNTIDx(F);
+  std::optional<unsigned> MaxNTIDy = getMaxNTIDy(F);
+  std::optional<unsigned> MaxNTIDz = getMaxNTIDz(F);
+  if (MaxNTIDx || MaxNTIDy || MaxNTIDz)
+    return MaxNTIDx.value_or(1) * MaxNTIDy.value_or(1) * MaxNTIDz.value_or(1);
+  return std::nullopt;
 }
 
 bool getMaxClusterRank(const Function &F, unsigned &x) {
   return findOneNVVMAnnotation(&F, "maxclusterrank", x);
 }
 
-bool getReqNTIDx(const Function &F, unsigned &x) {
-  return findOneNVVMAnnotation(&F, "reqntidx", x);
+std::optional<unsigned> getReqNTIDx(const Function &F) {
+  return findOneNVVMAnnotation(F, "reqntidx");
+}
+
+std::optional<unsigned> getReqNTIDy(const Function &F) {
+  return findOneNVVMAnnotation(F, "reqntidy");
 }
 
-bool getReqNTIDy(const Function &F, unsigned &y) {
-  return findOneNVVMAnnotation(&F, "reqntidy", y);
+std::optional<unsigned> getReqNTIDz(const Function &F) {
+  return findOneNVVMAnnotation(F, "reqntidz");
 }
 
-bool getReqNTIDz(const Function &F, unsigned &z) {
-  return findOneNVVMAnnotation(&F, "reqntidz", z);
+std::optional<unsigned> getReqNTID(const Function &F) {
+  // Note: The semantics here are a bit strange. See getMaxNTID.
+  std::optional<unsigned> ReqNTIDx = getReqNTIDx(F);
+  std::optional<unsigned> ReqNTIDy = getReqNTIDy(F);
+  std::optional<unsigned> ReqNTIDz = getReqNTIDz(F);
+  if (ReqNTIDx || ReqNTIDy || ReqNTIDz)
+    return ReqNTIDx.value_or(1) * ReqNTIDy.value_or(1) * ReqNTIDz.value_or(1);
+  return std::nullopt;
 }
 
 bool getMinCTASm(const Function &F, unsigned &x) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 2872db9fa2131448ad18cbca4085dabbab8d1f45..e020bc0f02e9606c8634b1e4c13fd2710b91e3d9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -48,13 +48,15 @@ std::string getTextureName(const Value &);
 std::string getSurfaceName(const Value &);
 std::string getSamplerName(const Value &);
 
-bool getMaxNTIDx(const Function &, unsigned &);
-bool getMaxNTIDy(const Function &, unsigned &);
-bool getMaxNTIDz(const Function &, unsigned &);
-
-bool getReqNTIDx(const Function &, unsigned &);
-bool getReqNTIDy(const Function &, unsigned &);
-bool getReqNTIDz(const Function &, unsigned &);
+std::optional<unsigned> getMaxNTIDx(const Function &);
+std::optional<unsigned> getMaxNTIDy(const Function &);
+std::optional<unsigned> getMaxNTIDz(const Function &);
+std::optional<unsigned> getMaxNTID(const Function &F);
+
+std::optional<unsigned> getReqNTIDx(const Function &);
+std::optional<unsigned> getReqNTIDy(const Function &);
+std::optional<unsigned> getReqNTIDz(const Function &);
+std::optional<unsigned> getReqNTID(const Function &);
 
 bool getMaxClusterRank(const Function &, unsigned &);
 bool getMinCTASm(const Function &, unsigned &);
diff --git a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp
index 5381646434eb8601896b84e216ef40c3ad83dcee..f9d21b38a7ecdb0e45589a31413206dc42adf472 100644
--- a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp
+++ b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp
@@ -1,4 +1,4 @@
-//===- NVVMIntrRange.cpp - Set !range metadata for NVVM intrinsics --------===//
+//===- NVVMIntrRange.cpp - Set range attributes for NVVM intrinsics -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,19 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This pass adds appropriate !range metadata for calls to NVVM
+// This pass adds appropriate range attributes for calls to NVVM
 // intrinsics that return a limited range of values.
 //
 //===----------------------------------------------------------------------===//
 
 #include "NVPTX.h"
-#include "llvm/IR/Constants.h"
+#include "NVPTXUtilities.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include <cstdint>
 
 using namespace llvm;
 
@@ -26,31 +28,20 @@ using namespace llvm;
 
 namespace llvm { void initializeNVVMIntrRangePass(PassRegistry &); }
 
-// Add !range metadata based on limits of given SM variant.
-static cl::opt<unsigned> NVVMIntrRangeSM("nvvm-intr-range-sm", cl::init(20),
-                                         cl::Hidden, cl::desc("SM variant"));
-
 namespace {
 class NVVMIntrRange : public FunctionPass {
- private:
-   unsigned SmVersion;
-
- public:
-   static char ID;
-   NVVMIntrRange() : NVVMIntrRange(NVVMIntrRangeSM) {}
-   NVVMIntrRange(unsigned int SmVersion)
-       : FunctionPass(ID), SmVersion(SmVersion) {
+public:
+  static char ID;
+  NVVMIntrRange() : FunctionPass(ID) {
 
-     initializeNVVMIntrRangePass(*PassRegistry::getPassRegistry());
-   }
+    initializeNVVMIntrRangePass(*PassRegistry::getPassRegistry());
+  }
 
-   bool runOnFunction(Function &) override;
+  bool runOnFunction(Function &) override;
 };
-}
+} // namespace
 
-FunctionPass *llvm::createNVVMIntrRangePass(unsigned int SmVersion) {
-  return new NVVMIntrRange(SmVersion);
-}
+FunctionPass *llvm::createNVVMIntrRangePass() { return new NVVMIntrRange(); }
 
 char NVVMIntrRange::ID = 0;
 INITIALIZE_PASS(NVVMIntrRange, "nvvm-intr-range",
@@ -58,112 +49,110 @@ INITIALIZE_PASS(NVVMIntrRange, "nvvm-intr-range",
 
 // Adds the passed-in [Low,High) range information as metadata to the
 // passed-in call instruction.
-static bool addRangeMetadata(uint64_t Low, uint64_t High, CallInst *C) {
-  // This call already has range metadata, nothing to do.
-  if (C->getMetadata(LLVMContext::MD_range))
+static bool addRangeAttr(uint64_t Low, uint64_t High, IntrinsicInst *II) {
+  if (II->getMetadata(LLVMContext::MD_range))
     return false;
 
-  LLVMContext &Context = C->getParent()->getContext();
-  IntegerType *Int32Ty = Type::getInt32Ty(Context);
-  Metadata *LowAndHigh[] = {
-      ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Low)),
-      ConstantAsMetadata::get(ConstantInt::get(Int32Ty, High))};
-  C->setMetadata(LLVMContext::MD_range, MDNode::get(Context, LowAndHigh));
+  const uint64_t BitWidth = II->getType()->getIntegerBitWidth();
+  ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High));
+
+  if (auto CurrentRange = II->getRange())
+    Range = Range.intersectWith(CurrentRange.value());
+
+  II->addRangeRetAttr(Range);
   return true;
 }
 
-static bool runNVVMIntrRange(Function &F, unsigned SmVersion) {
+static bool runNVVMIntrRange(Function &F) {
   struct {
     unsigned x, y, z;
   } MaxBlockSize, MaxGridSize;
-  MaxBlockSize.x = 1024;
-  MaxBlockSize.y = 1024;
-  MaxBlockSize.z = 64;
 
-  MaxGridSize.x = SmVersion >= 30 ? 0x7fffffff : 0xffff;
+  const unsigned MetadataNTID = getReqNTID(F).value_or(
+      getMaxNTID(F).value_or(std::numeric_limits<unsigned>::max()));
+
+  MaxBlockSize.x = std::min(1024u, MetadataNTID);
+  MaxBlockSize.y = std::min(1024u, MetadataNTID);
+  MaxBlockSize.z = std::min(64u, MetadataNTID);
+
+  MaxGridSize.x = 0x7fffffff;
   MaxGridSize.y = 0xffff;
   MaxGridSize.z = 0xffff;
 
   // Go through the calls in this function.
   bool Changed = false;
   for (Instruction &I : instructions(F)) {
-    CallInst *Call = dyn_cast<CallInst>(&I);
-    if (!Call)
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
+    if (!II)
       continue;
 
-    if (Function *Callee = Call->getCalledFunction()) {
-      switch (Callee->getIntrinsicID()) {
-      // Index within block
-      case Intrinsic::nvvm_read_ptx_sreg_tid_x:
-        Changed |= addRangeMetadata(0, MaxBlockSize.x, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_tid_y:
-        Changed |= addRangeMetadata(0, MaxBlockSize.y, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_tid_z:
-        Changed |= addRangeMetadata(0, MaxBlockSize.z, Call);
-        break;
-
-      // Block size
-      case Intrinsic::nvvm_read_ptx_sreg_ntid_x:
-        Changed |= addRangeMetadata(1, MaxBlockSize.x+1, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_ntid_y:
-        Changed |= addRangeMetadata(1, MaxBlockSize.y+1, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_ntid_z:
-        Changed |= addRangeMetadata(1, MaxBlockSize.z+1, Call);
-        break;
-
-      // Index within grid
-      case Intrinsic::nvvm_read_ptx_sreg_ctaid_x:
-        Changed |= addRangeMetadata(0, MaxGridSize.x, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_ctaid_y:
-        Changed |= addRangeMetadata(0, MaxGridSize.y, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_ctaid_z:
-        Changed |= addRangeMetadata(0, MaxGridSize.z, Call);
-        break;
-
-      // Grid size
-      case Intrinsic::nvvm_read_ptx_sreg_nctaid_x:
-        Changed |= addRangeMetadata(1, MaxGridSize.x+1, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_nctaid_y:
-        Changed |= addRangeMetadata(1, MaxGridSize.y+1, Call);
-        break;
-      case Intrinsic::nvvm_read_ptx_sreg_nctaid_z:
-        Changed |= addRangeMetadata(1, MaxGridSize.z+1, Call);
-        break;
-
-      // warp size is constant 32.
-      case Intrinsic::nvvm_read_ptx_sreg_warpsize:
-        Changed |= addRangeMetadata(32, 32+1, Call);
-        break;
-
-      // Lane ID is [0..warpsize)
-      case Intrinsic::nvvm_read_ptx_sreg_laneid:
-        Changed |= addRangeMetadata(0, 32, Call);
-        break;
-
-      default:
-        break;
-      }
+    switch (II->getIntrinsicID()) {
+    // Index within block
+    case Intrinsic::nvvm_read_ptx_sreg_tid_x:
+      Changed |= addRangeAttr(0, MaxBlockSize.x, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_tid_y:
+      Changed |= addRangeAttr(0, MaxBlockSize.y, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_tid_z:
+      Changed |= addRangeAttr(0, MaxBlockSize.z, II);
+      break;
+
+    // Block size
+    case Intrinsic::nvvm_read_ptx_sreg_ntid_x:
+      Changed |= addRangeAttr(1, MaxBlockSize.x + 1, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_ntid_y:
+      Changed |= addRangeAttr(1, MaxBlockSize.y + 1, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_ntid_z:
+      Changed |= addRangeAttr(1, MaxBlockSize.z + 1, II);
+      break;
+
+    // Index within grid
+    case Intrinsic::nvvm_read_ptx_sreg_ctaid_x:
+      Changed |= addRangeAttr(0, MaxGridSize.x, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_ctaid_y:
+      Changed |= addRangeAttr(0, MaxGridSize.y, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_ctaid_z:
+      Changed |= addRangeAttr(0, MaxGridSize.z, II);
+      break;
+
+    // Grid size
+    case Intrinsic::nvvm_read_ptx_sreg_nctaid_x:
+      Changed |= addRangeAttr(1, MaxGridSize.x + 1, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_nctaid_y:
+      Changed |= addRangeAttr(1, MaxGridSize.y + 1, II);
+      break;
+    case Intrinsic::nvvm_read_ptx_sreg_nctaid_z:
+      Changed |= addRangeAttr(1, MaxGridSize.z + 1, II);
+      break;
+
+    // warp size is constant 32.
+    case Intrinsic::nvvm_read_ptx_sreg_warpsize:
+      Changed |= addRangeAttr(32, 32 + 1, II);
+      break;
+
+    // Lane ID is [0..warpsize)
+    case Intrinsic::nvvm_read_ptx_sreg_laneid:
+      Changed |= addRangeAttr(0, 32, II);
+      break;
+
+    default:
+      break;
     }
   }
 
   return Changed;
 }
 
-bool NVVMIntrRange::runOnFunction(Function &F) {
-  return runNVVMIntrRange(F, SmVersion);
-}
-
-NVVMIntrRangePass::NVVMIntrRangePass() : NVVMIntrRangePass(NVVMIntrRangeSM) {}
+bool NVVMIntrRange::runOnFunction(Function &F) { return runNVVMIntrRange(F); }
 
 PreservedAnalyses NVVMIntrRangePass::run(Function &F,
                                          FunctionAnalysisManager &AM) {
-  return runNVVMIntrRange(F, SmVersion) ? PreservedAnalyses::none()
-                                        : PreservedAnalyses::all();
+  return runNVVMIntrRange(F) ? PreservedAnalyses::none()
+                             : PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index f4e84ade3b5ac91b508b1393121c064fb7f5689f..bc0ae7a32c05171f2a0a7611db06dc087aef68a8 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1079,13 +1079,13 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     assert(IsAIX && TM.getCodeModel() == CodeModel::Small &&
            "PseudoOp only valid for small code model AIX");
 
-    // Transform %rN = ADDItoc/8 @op1, %r2.
+    // Transform %rN = ADDItoc/8 %r2, @op1.
     LowerPPCMachineInstrToMCInst(MI, TmpInst, *this);
 
     // Change the opcode to load address.
     TmpInst.setOpcode((!IsPPC64) ? (PPC::LA) : (PPC::LA8));
 
-    const MachineOperand &MO = MI->getOperand(1);
+    const MachineOperand &MO = MI->getOperand(2);
     assert(MO.isGlobal() && "Invalid operand for ADDItoc[8].");
 
     // Map the operand to its corresponding MCSymbol.
@@ -1094,7 +1094,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MCExpr *Exp =
         MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_None, OutContext);
 
-    TmpInst.getOperand(1) = TmpInst.getOperand(2);
     TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
     return;
diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
index 735050641adff91da8a1074060b315737b07dc46..a07954bd0d8b34c93e25fad797bd7642d960b46f 100644
--- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
@@ -2080,13 +2080,15 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) {
                       cast<GlobalVariable>(GV)->hasAttribute("toc-data");
 
   // For small code model, generate a simple TOC load.
-  if (CModel == CodeModel::Small)
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-            IsAIXTocData ? TII.get(PPC::ADDItoc8) : TII.get(PPC::LDtoc),
-            DestReg)
-        .addGlobalAddress(GV)
-        .addReg(PPC::X2);
-  else {
+  if (CModel == CodeModel::Small) {
+    auto MIB = BuildMI(
+        *FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
+        IsAIXTocData ? TII.get(PPC::ADDItoc8) : TII.get(PPC::LDtoc), DestReg);
+    if (IsAIXTocData)
+      MIB.addReg(PPC::X2).addGlobalAddress(GV);
+    else
+      MIB.addGlobalAddress(GV).addReg(PPC::X2);
+  } else {
     // If the address is an externally defined symbol, a symbol with common
     // or externally available linkage, a non-local function address, or a
     // jump table address (not yet needed), or if we are generating code
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 8444266459c4336af8905135ae26a075e6e38045..201b2d162372b8da11b6913f4236d4c8fd4757ce 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -317,7 +317,8 @@ PPCFrameLowering::determineFrameLayout(const MachineFunction &MF,
                        !MFI.adjustsStack() &&       // No calls.
                        !MustSaveLR(MF, LR) &&       // No need to save LR.
                        !FI->mustSaveTOC() &&        // No need to save TOC.
-                       !RegInfo->hasBasePointer(MF); // No special alignment.
+                       !RegInfo->hasBasePointer(MF) && // No special alignment.
+                       !MFI.isFrameAddressTaken();
 
   // Note: for PPC32 SVR4ABI, we can still generate stackless
   // code if all local vars are reg-allocated.
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 275b3337a2763fc66757b693a0e3dc7534faa4a1..1a69d1e89313b82371cfe546381f5c2a19f72d2d 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -6102,8 +6102,15 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
                                    EVT OperandTy) {
       SDValue GA = TocEntry->getOperand(0);
       SDValue TocBase = TocEntry->getOperand(1);
-      SDNode *MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase);
-      transferMemOperands(TocEntry, MN);
+      SDNode *MN = nullptr;
+      if (OpCode == PPC::ADDItoc || OpCode == PPC::ADDItoc8)
+        // toc-data access doesn't involve in loading from got, no need to
+        // keep memory operands.
+        MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, TocBase, GA);
+      else {
+        MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase);
+        transferMemOperands(TocEntry, MN);
+      }
       ReplaceNode(TocEntry, MN);
     };
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 9af8ada7837614c1d311bc555ab5460a3f53e609..eda5eb975e700a463fb94583da9636fb4134e4cd 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -1485,11 +1485,9 @@ def ADDItocL8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:
 }
 
 // Local Data Transform
-def ADDItoc8 : PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc_nox0:$reg),
+def ADDItoc8 : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp),
                    "#ADDItoc8",
-                   [(set i64:$rD,
-                     (PPCtoc_entry tglobaladdr:$disp, i64:$reg))]>, isPPC64;
-
+                   []>, isPPC64;
 let mayLoad = 1 in
 def LDtocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc_nox0:$reg),
                    "#LDtocL", []>, isPPC64;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index df6b2bf1a7b7701a5c1b3a4e947f4307f292da50..09f829943528c4e281c19a3faf770a1193177786 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -3345,10 +3345,8 @@ def LWZtocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc_nor
 def ADDIStocHA : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp),
                        "#ADDIStocHA", []>;
 // TOC Data Transform on AIX
-def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc:$reg),
-                   "#ADDItoc",
-                   [(set i32:$rD,
-                     (PPCtoc_entry tglobaladdr:$disp, i32:$reg))]>;
+def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc:$reg, tocentry32:$disp),
+                   "#ADDItoc", []>;
 def ADDItocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp),
                    "#ADDItocL", []>;
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index d92998ced91efff618bbd296fafcf52516a45460..5906a2cdb3bfa1bbdc0630cbf26744aa7cc3a683 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1245,7 +1245,7 @@ public:
   }
 
   void addRegRegOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
+    assert(N == 2 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::createReg(RegReg.Reg1));
     Inst.addOperand(MCOperand::createReg(RegReg.Reg2));
   }
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index c18892ac62f247f60b50200d474e18fb47b2af70..beee9405de02ac2fe081efc15a6adb5e726368eb 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -340,7 +340,7 @@ static bool isSupportedArgumentType(Type *T, const RISCVSubtarget &Subtarget,
   // supported yet.
   if (T->isIntegerTy())
     return T->getIntegerBitWidth() <= Subtarget.getXLen() * 2;
-  if (T->isFloatTy() || T->isDoubleTy())
+  if (T->isHalfTy() || T->isFloatTy() || T->isDoubleTy())
     return true;
   if (T->isPointerTy())
     return true;
@@ -361,7 +361,7 @@ static bool isSupportedReturnType(Type *T, const RISCVSubtarget &Subtarget,
   // supported yet.
   if (T->isIntegerTy())
     return T->getIntegerBitWidth() <= Subtarget.getXLen() * 2;
-  if (T->isFloatTy() || T->isDoubleTy())
+  if (T->isHalfTy() || T->isFloatTy() || T->isDoubleTy())
     return true;
   if (T->isPointerTy())
     return true;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index da8daa573b89b3210bcc97cbcf545431aab7030c..a091380e8ce82c9504e1571269b22a23842a1c1f 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -849,6 +849,8 @@ const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
   }
 
   if (RB.getID() == RISCV::FPRBRegBankID) {
+    if (Ty.getSizeInBits() == 16)
+      return &RISCV::FPR16RegClass;
     if (Ty.getSizeInBits() == 32)
       return &RISCV::FPR32RegClass;
     if (Ty.getSizeInBits() == 64)
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index dbfcab7233bf8d6443740359c54e0d6d0e9dd655..816e62ea24edb99b907ec40d99cbecc251bd4528 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -409,7 +409,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
       .legalIf(all(typeInSet(0, {s32, sXLen}), typeIsScalarFPArith(1, ST)))
       .widenScalarToNextPow2(0)
-      .clampScalar(0, s32, sXLen);
+      .clampScalar(0, s32, sXLen)
+      .libcall();
 
   getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
       .legalIf(all(typeIsScalarFPArith(0, ST), typeInSet(1, {s32, sXLen})))
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 251401db3e28142812be23522ae568d4a38608ba..8c1f8dca4e102bfb56d44ff82234d7a7ad039231 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -1527,6 +1527,67 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
   case ISD::LOAD: {
     if (tryIndexedLoad(Node))
       return;
+
+    if (Subtarget->hasVendorXCVmem()) {
+      // We match post-incrementing load here
+      LoadSDNode *Load = cast<LoadSDNode>(Node);
+      if (Load->getAddressingMode() != ISD::POST_INC)
+        break;
+
+      SDValue Chain = Node->getOperand(0);
+      SDValue Base = Node->getOperand(1);
+      SDValue Offset = Node->getOperand(2);
+
+      bool Simm12 = false;
+      bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
+
+      if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Offset)) {
+        int ConstantVal = ConstantOffset->getSExtValue();
+        Simm12 = isInt<12>(ConstantVal);
+        if (Simm12)
+          Offset = CurDAG->getTargetConstant(ConstantVal, SDLoc(Offset),
+                                             Offset.getValueType());
+      }
+
+      unsigned Opcode = 0;
+      switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
+      case MVT::i8:
+        if (Simm12 && SignExtend)
+          Opcode = RISCV::CV_LB_ri_inc;
+        else if (Simm12 && !SignExtend)
+          Opcode = RISCV::CV_LBU_ri_inc;
+        else if (!Simm12 && SignExtend)
+          Opcode = RISCV::CV_LB_rr_inc;
+        else
+          Opcode = RISCV::CV_LBU_rr_inc;
+        break;
+      case MVT::i16:
+        if (Simm12 && SignExtend)
+          Opcode = RISCV::CV_LH_ri_inc;
+        else if (Simm12 && !SignExtend)
+          Opcode = RISCV::CV_LHU_ri_inc;
+        else if (!Simm12 && SignExtend)
+          Opcode = RISCV::CV_LH_rr_inc;
+        else
+          Opcode = RISCV::CV_LHU_rr_inc;
+        break;
+      case MVT::i32:
+        if (Simm12)
+          Opcode = RISCV::CV_LW_ri_inc;
+        else
+          Opcode = RISCV::CV_LW_rr_inc;
+        break;
+      default:
+        break;
+      }
+      if (!Opcode)
+        break;
+
+      ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, XLenVT, XLenVT,
+                                               Chain.getSimpleValueType(), Base,
+                                               Offset, Chain));
+      return;
+    }
     break;
   }
   case ISD::INTRINSIC_WO_CHAIN: {
@@ -2669,6 +2730,19 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
   return true;
 }
 
+bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
+                                         SDValue &Offset) {
+  if (Addr.getOpcode() != ISD::ADD)
+    return false;
+
+  if (isa<ConstantSDNode>(Addr.getOperand(1)))
+    return false;
+
+  Base = Addr.getOperand(1);
+  Offset = Addr.getOperand(0);
+  return true;
+}
+
 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
                                         SDValue &ShAmt) {
   ShAmt = N;
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 5d70245c0c5570a143ccaa2fefe48d9d2081f0bd..1b3b00eeccce8b20688b2b736d01e053b628f2fe 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -78,6 +78,8 @@ public:
     return false;
   }
 
+  bool SelectAddrRegReg(SDValue Addr, SDValue &Base, SDValue &Offset);
+
   bool tryShrinkShlLogicImm(SDNode *Node);
   bool trySignedBitfieldExtract(SDNode *Node);
   bool tryIndexedLoad(SDNode *Node);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 8ace5d79af079b096f7a0579cc8cf9183edd27b5..af3950773e4d0e2ab345807ff1753b0f7877695f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1102,6 +1102,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                             ISD::EXTRACT_SUBVECTOR},
                            VT, Custom);
         setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
+        if (Subtarget.hasStdExtZfbfmin()) {
+          if (Subtarget.hasVInstructionsF16())
+            setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
+          else if (Subtarget.hasVInstructionsF16Minimal())
+            setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
+        }
+        setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
+                           Custom);
+        setOperationAction(ISD::SELECT_CC, VT, Expand);
         // TODO: Promote to fp32.
       }
     }
@@ -1331,6 +1340,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                               ISD::EXTRACT_SUBVECTOR},
                              VT, Custom);
           setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
+          if (Subtarget.hasStdExtZfbfmin()) {
+            if (Subtarget.hasVInstructionsF16())
+              setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
+            else if (Subtarget.hasVInstructionsF16Minimal())
+              setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
+          }
+          setOperationAction(
+              {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
+              Custom);
           // TODO: Promote to fp32.
           continue;
         }
@@ -1433,6 +1451,16 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     }
   }
 
+  if (Subtarget.hasVendorXCVmem()) {
+    setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
+    setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
+    setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
+
+    setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
+    setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
+    setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
+  }
+
   // Function alignments.
   const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4);
   setMinFunctionAlignment(FunctionAlignment);
@@ -1452,7 +1480,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.hasStdExtZbb())
     setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN});
 
-  if (Subtarget.hasStdExtZbs() && Subtarget.is64Bit())
+  if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
+      Subtarget.hasStdExtV())
     setTargetDAGCombine(ISD::TRUNCATE);
 
   if (Subtarget.hasStdExtZbkb())
@@ -6704,10 +6733,16 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::BUILD_VECTOR:
     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
   case ISD::SPLAT_VECTOR:
-    if (Op.getValueType().getScalarType() == MVT::f16 &&
-        (Subtarget.hasVInstructionsF16Minimal() &&
-         !Subtarget.hasVInstructionsF16())) {
-      if (Op.getValueType() == MVT::nxv32f16)
+    if ((Op.getValueType().getScalarType() == MVT::f16 &&
+         (Subtarget.hasVInstructionsF16Minimal() &&
+          Subtarget.hasStdExtZfhminOrZhinxmin() &&
+          !Subtarget.hasVInstructionsF16())) ||
+        (Op.getValueType().getScalarType() == MVT::bf16 &&
+         (Subtarget.hasVInstructionsBF16() && Subtarget.hasStdExtZfbfmin() &&
+          Subtarget.hasVInstructionsF16Minimal() &&
+          !Subtarget.hasVInstructionsF16()))) {
+      if (Op.getValueType() == MVT::nxv32f16 ||
+          Op.getValueType() == MVT::nxv32bf16)
         return SplitVectorOp(Op, DAG);
       SDLoc DL(Op);
       SDValue NewScalar =
@@ -13370,6 +13405,76 @@ static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
   return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
 }
 
+// Fold (vXi8 (trunc (vselect (setltu, X, 256), X, (sext (setgt X, 0))))) to
+// (vXi8 (trunc (smin (smax X, 0), 255))). This represents saturating a signed
+// value to an unsigned value. This will be lowered to vmax and series of
+// vnclipu instructions later. This can be extended to other truncated types
+// other than i8 by replacing 256 and 255 with the equivalent constants for the
+// type.
+static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0);
+  EVT SrcVT = N0.getValueType();
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(SrcVT))
+    return SDValue();
+
+  if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
+    return SDValue();
+
+  SDValue Cond = N0.getOperand(0);
+  SDValue True = N0.getOperand(1);
+  SDValue False = N0.getOperand(2);
+
+  if (Cond.getOpcode() != ISD::SETCC)
+    return SDValue();
+
+  // FIXME: Support the version of this pattern with the select operands
+  // swapped.
+  ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+  if (CCVal != ISD::SETULT)
+    return SDValue();
+
+  SDValue CondLHS = Cond.getOperand(0);
+  SDValue CondRHS = Cond.getOperand(1);
+
+  if (CondLHS != True)
+    return SDValue();
+
+  unsigned ScalarBits = VT.getScalarSizeInBits();
+
+  // FIXME: Support other constants.
+  ConstantSDNode *CondRHSC = isConstOrConstSplat(CondRHS);
+  if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
+    return SDValue();
+
+  if (False.getOpcode() != ISD::SIGN_EXTEND)
+    return SDValue();
+
+  False = False.getOperand(0);
+
+  if (False.getOpcode() != ISD::SETCC || False.getOperand(0) != True)
+    return SDValue();
+
+  ConstantSDNode *FalseRHSC = isConstOrConstSplat(False.getOperand(1));
+  if (!FalseRHSC || !FalseRHSC->isZero())
+    return SDValue();
+
+  ISD::CondCode CCVal2 = cast<CondCodeSDNode>(False.getOperand(2))->get();
+  if (CCVal2 != ISD::SETGT)
+    return SDValue();
+
+  // Emit the signed to unsigned saturation pattern.
+  SDLoc DL(N);
+  SDValue Max =
+      DAG.getNode(ISD::SMAX, DL, SrcVT, True, DAG.getConstant(0, DL, SrcVT));
+  SDValue Min =
+      DAG.getNode(ISD::SMIN, DL, SrcVT, Max,
+                  DAG.getConstant((1ULL << ScalarBits) - 1, DL, SrcVT));
+  return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
+}
+
 static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
   SDValue N0 = N->getOperand(0);
@@ -13390,7 +13495,7 @@ static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
   }
 
-  return SDValue();
+  return combineTruncSelectToSMaxUSat(N, DAG);
 }
 
 // Combines two comparison operation and logic operation to one selection
@@ -20909,6 +21014,26 @@ bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
                                                      SDValue &Offset,
                                                      ISD::MemIndexedMode &AM,
                                                      SelectionDAG &DAG) const {
+  if (Subtarget.hasVendorXCVmem()) {
+    if (Op->getOpcode() != ISD::ADD)
+      return false;
+
+    if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N))
+      Base = LS->getBasePtr();
+    else
+      return false;
+
+    if (Base == Op->getOperand(0))
+      Offset = Op->getOperand(1);
+    else if (Base == Op->getOperand(1))
+      Offset = Op->getOperand(0);
+    else
+      return false;
+
+    AM = ISD::POST_INC;
+    return true;
+  }
+
   EVT VT;
   SDValue Ptr;
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index a96768240a9336dd77bd351eaeb99577c675411e..82358cdd45edc6349fd16f317c766de3b74efd39 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -932,11 +932,11 @@ RISCVInsertVSETVLI::getInfoForVSETVLI(const MachineInstr &MI) const {
            "Can't handle X0, X0 vsetvli yet");
     if (AVLReg == RISCV::X0)
       NewInfo.setAVLVLMAX();
-    else if (VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS))
-      NewInfo.setAVLRegDef(VNI, AVLReg);
-    else {
-      assert(MI.getOperand(1).isUndef());
+    else if (MI.getOperand(1).isUndef())
       NewInfo.setAVLIgnored();
+    else {
+      VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS);
+      NewInfo.setAVLRegDef(VNI, AVLReg);
     }
   }
   NewInfo.setVTYPE(MI.getOperand(2).getImm());
@@ -1008,11 +1008,11 @@ RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const {
       }
       else
         InstrInfo.setAVLImm(Imm);
-    } else if (VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS)) {
-      InstrInfo.setAVLRegDef(VNI, VLOp.getReg());
-    } else {
-      assert(VLOp.isUndef());
+    } else if (VLOp.isUndef()) {
       InstrInfo.setAVLIgnored();
+    } else {
+      VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS);
+      InstrInfo.setAVLRegDef(VNI, VLOp.getReg());
     }
   } else {
     assert(isScalarExtractInstr(MI));
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index a1b078910e29c98c3f05c89a9671fcfa9b77ebc9..4cdf08a46f285b570fcbe4a08c91d1e692dff3c9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -943,6 +943,9 @@ def : InstAlias<"ret",                   (JALR      X0,      X1, 0), 4>;
 def : InstAlias<"jr $rs, $offset",        (JALR      X0, GPR:$rs, simm12:$offset), 0>;
 def : InstAlias<"jalr $rs, $offset",      (JALR      X1, GPR:$rs, simm12:$offset), 0>;
 def : InstAlias<"jalr $rd, $rs, $offset", (JALR GPR:$rd, GPR:$rs, simm12:$offset), 0>;
+def : InstAlias<"jr (${rs})",             (JALR      X0, GPR:$rs, 0), 0>;
+def : InstAlias<"jalr (${rs})",           (JALR      X1, GPR:$rs, 0), 0>;
+def : InstAlias<"jalr $rd, (${rs})",      (JALR GPR:$rd, GPR:$rs, 0), 0>;
 
 def : InstAlias<"fence", (FENCE 0xF, 0xF)>; // 0xF == iorw
 
@@ -1140,6 +1143,9 @@ def : InstAlias<".insn_i $opcode, $funct3, $rd, $rs1, $imm12",
 def : InstAlias<".insn_i $opcode, $funct3, $rd, ${imm12}(${rs1})",
                 (InsnI_Mem AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3,
                            AnyReg:$rs1, simm12:$imm12)>;
+def : InstAlias<".insn_i $opcode, $funct3, $rd, (${rs1})",
+                (InsnI_Mem AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3,
+                           AnyReg:$rs1, 0)>;
 def : InstAlias<".insn_b $opcode, $funct3, $rs1, $rs2, $imm12",
                 (InsnB uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1,
                        AnyReg:$rs2, simm13_lsb0:$imm12)>;
@@ -1157,6 +1163,9 @@ def : InstAlias<".insn_uj $opcode, $rd, $imm20",
 def : InstAlias<".insn_s $opcode, $funct3, $rs2, ${imm12}(${rs1})",
                 (InsnS uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs2,
                        AnyReg:$rs1, simm12:$imm12)>;
+def : InstAlias<".insn_s $opcode, $funct3, $rs2, (${rs1})",
+                (InsnS uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs2,
+                       AnyReg:$rs1, 0)>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index f4e50d7aa45c5b5bcee5005b5e14d227805ad68f..458d081763e931a227709aa3d3445fd92674ff17 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -823,9 +823,15 @@ def : InstAlias<".insn_css $opcode, $funct3, $rs2, $imm6",
 def : InstAlias<".insn_cl $opcode, $funct3, $rd, ${imm5}(${rs1})",
                 (InsnCL AnyRegC:$rd, uimm2_opcode:$opcode, uimm3:$funct3,
                         AnyRegC:$rs1, uimm5:$imm5)>;
+def : InstAlias<".insn_cl $opcode, $funct3, $rd, (${rs1})",
+                (InsnCL AnyRegC:$rd, uimm2_opcode:$opcode, uimm3:$funct3,
+                        AnyRegC:$rs1, 0)>;
 def : InstAlias<".insn_cs $opcode, $funct3, $rs2, ${imm5}(${rs1})",
                 (InsnCS uimm2_opcode:$opcode, uimm3:$funct3, AnyRegC:$rs2,
                         AnyRegC:$rs1, uimm5:$imm5)>;
+def : InstAlias<".insn_cs $opcode, $funct3, $rs2, (${rs1})",
+                (InsnCS uimm2_opcode:$opcode, uimm3:$funct3, AnyRegC:$rs2,
+                        AnyRegC:$rs1, 0)>;
 def : InstAlias<".insn_ca $opcode, $funct6, $funct2, $rd, $rs2",
                 (InsnCA AnyRegC:$rd, uimm2_opcode:$opcode, uimm6:$funct6,
                         uimm2:$funct2, AnyRegC:$rs2)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 5fe5a7a5bd5cce57c23398dfae9886ffa3087d0a..77d3644687fddd5d9f764e40436f66383f568b32 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -382,7 +382,20 @@ class GetIntVTypeInfo<VTypeInfo vti> {
   // Equivalent integer vector type. Eg.
   //   VI8M1 → VI8M1 (identity)
   //   VF64M4 → VI64M4
-  VTypeInfo Vti = !cast<VTypeInfo>(!subst("VF", "VI", !cast<string>(vti)));
+  VTypeInfo Vti = !cast<VTypeInfo>(!subst("VBF", "VI",
+                                          !subst("VF", "VI",
+                                                 !cast<string>(vti))));
+}
+
+// This functor is used to obtain the fp vector type that has the same SEW and
+// multiplier as the input parameter type.
+class GetFpVTypeInfo<VTypeInfo vti> {
+  // Equivalent integer vector type. Eg.
+  //   VF16M1 → VF16M1 (identity)
+  //   VBF16M1 → VF16M1
+  VTypeInfo Vti = !cast<VTypeInfo>(!subst("VBF", "VF",
+                                          !subst("VI", "VF",
+                                                 !cast<string>(vti))));
 }
 
 class MTypeInfo<ValueType Mas, LMULInfo M, string Bx> {
@@ -1033,6 +1046,22 @@ class VPseudoUnaryNoMask<DAGOperand RetClass,
   let HasVecPolicyOp = 1;
 }
 
+class VPseudoUnaryNoMaskNoPolicy<DAGOperand RetClass,
+                                 DAGOperand OpClass,
+                                 string Constraint = "",
+                                 int TargetConstraintType = 1> :
+      Pseudo<(outs RetClass:$rd),
+             (ins OpClass:$rs2, AVL:$vl, ixlenimm:$sew), []>,
+      RISCVVPseudo {
+  let mayLoad = 0;
+  let mayStore = 0;
+  let hasSideEffects = 0;
+  let Constraints = Constraint;
+  let TargetOverlapConstraintType = TargetConstraintType;
+  let HasVLOp = 1;
+  let HasSEWOp = 1;
+}
+
 class VPseudoUnaryNoMaskRoundingMode<DAGOperand RetClass,
                                      DAGOperand OpClass,
                                      string Constraint = "",
@@ -1205,11 +1234,11 @@ class VPseudoBinaryNoMask<VReg RetClass,
   let HasSEWOp = 1;
 }
 
-class VPseudoBinaryNoMaskTU<VReg RetClass,
-                            VReg Op1Class,
-                            DAGOperand Op2Class,
-                            string Constraint,
-                            int TargetConstraintType = 1> :
+class VPseudoBinaryNoMaskPolicy<VReg RetClass,
+                                VReg Op1Class,
+                                DAGOperand Op2Class,
+                                string Constraint,
+                                int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1, AVL:$vl,
                   ixlenimm:$sew, ixlenimm:$policy), []>,
@@ -1344,23 +1373,6 @@ class VPseudoIStoreMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL,
   let HasSEWOp = 1;
 }
 
-class VPseudoBinaryMask<VReg RetClass,
-                        RegisterClass Op1Class,
-                        DAGOperand Op2Class,
-                        string Constraint> :
-      Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
-             (ins GetVRegNoV0<RetClass>.R:$merge,
-                  Op1Class:$rs2, Op2Class:$rs1,
-                  VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
-      RISCVVPseudo {
-  let mayLoad = 0;
-  let mayStore = 0;
-  let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-}
-
 class VPseudoBinaryMaskPolicy<VReg RetClass,
                               RegisterClass Op1Class,
                               DAGOperand Op2Class,
@@ -1384,8 +1396,7 @@ class VPseudoBinaryMaskPolicy<VReg RetClass,
 
 class VPseudoTernaryMaskPolicy<VReg RetClass,
                                RegisterClass Op1Class,
-                               DAGOperand Op2Class,
-                               string Constraint> :
+                               DAGOperand Op2Class> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$merge,
                   Op1Class:$rs2, Op2Class:$rs1,
@@ -1394,7 +1405,7 @@ class VPseudoTernaryMaskPolicy<VReg RetClass,
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
+  let Constraints = "$rd = $merge";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -1402,8 +1413,7 @@ class VPseudoTernaryMaskPolicy<VReg RetClass,
 
 class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
                                            RegisterClass Op1Class,
-                                           DAGOperand Op2Class,
-                                           string Constraint> :
+                                           DAGOperand Op2Class> :
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
              (ins GetVRegNoV0<RetClass>.R:$merge,
                   Op1Class:$rs2, Op2Class:$rs1,
@@ -1414,7 +1424,7 @@ class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
+  let Constraints = "$rd = $merge";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -1422,25 +1432,7 @@ class VPseudoTernaryMaskPolicyRoundingMode<VReg RetClass,
   let UsesVXRM = 0;
 }
 
-// Like VPseudoBinaryNoMask, but output can be V0.
-class VPseudoBinaryMOutNoMask<VReg RetClass,
-                              VReg Op1Class,
-                              DAGOperand Op2Class,
-                              string Constraint,
-                              int TargetConstraintType = 1> :
-      Pseudo<(outs RetClass:$rd),
-             (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>,
-      RISCVVPseudo {
-  let mayLoad = 0;
-  let mayStore = 0;
-  let hasSideEffects = 0;
-  let Constraints = Constraint;
-  let TargetOverlapConstraintType = TargetConstraintType;
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-}
-
-// Like VPseudoBinaryMask, but output can be V0.
+// Like VPseudoBinaryMaskPolicy, but output can be V0 and there is no policy.
 class VPseudoBinaryMOutMask<VReg RetClass,
                             RegisterClass Op1Class,
                             DAGOperand Op2Class,
@@ -1461,8 +1453,8 @@ class VPseudoBinaryMOutMask<VReg RetClass,
   let UsesMaskPolicy = 1;
 }
 
-// Special version of VPseudoBinaryMask where we pretend the first source is
-// tied to the destination so we can workaround the earlyclobber constraint.
+// Special version of VPseudoBinaryMaskPolicy where we pretend the first source
+// is tied to the destination so we can workaround the earlyclobber constraint.
 // This allows maskedoff and rs2 to be the same register.
 class VPseudoTiedBinaryMask<VReg RetClass,
                             DAGOperand Op2Class,
@@ -1510,13 +1502,13 @@ class VPseudoTiedBinaryMaskRoundingMode<VReg RetClass,
   let UsesVXRM = 0;
 }
 
-class VPseudoBinaryCarryIn<VReg RetClass,
-                           VReg Op1Class,
-                           DAGOperand Op2Class,
-                           LMULInfo MInfo,
-                           bit CarryIn,
-                           string Constraint,
-                           int TargetConstraintType = 1> :
+class VPseudoBinaryCarry<VReg RetClass,
+                         VReg Op1Class,
+                         DAGOperand Op2Class,
+                         LMULInfo MInfo,
+                         bit CarryIn,
+                         string Constraint,
+                         int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              !if(CarryIn,
                 (ins Op1Class:$rs2, Op2Class:$rs1,
@@ -1538,20 +1530,15 @@ class VPseudoTiedBinaryCarryIn<VReg RetClass,
                                VReg Op1Class,
                                DAGOperand Op2Class,
                                LMULInfo MInfo,
-                               bit CarryIn,
-                               string Constraint,
                                int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
-             !if(CarryIn,
-                (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1,
-                     VMV0:$carry, AVL:$vl, ixlenimm:$sew),
-                (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1,
-                     AVL:$vl, ixlenimm:$sew)), []>,
+             (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1,
+                  VMV0:$carry, AVL:$vl, ixlenimm:$sew), []>,
       RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
+  let Constraints = "$rd = $merge";
   let TargetOverlapConstraintType = TargetConstraintType;
   let HasVLOp = 1;
   let HasSEWOp = 1;
@@ -1578,7 +1565,7 @@ class VPseudoTernaryNoMask<VReg RetClass,
 class VPseudoTernaryNoMaskWithPolicy<VReg RetClass,
                                      RegisterClass Op1Class,
                                      DAGOperand Op2Class,
-                                     string Constraint,
+                                     string Constraint = "",
                                      int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
@@ -1597,7 +1584,7 @@ class VPseudoTernaryNoMaskWithPolicy<VReg RetClass,
 class VPseudoTernaryNoMaskWithPolicyRoundingMode<VReg RetClass,
                                                  RegisterClass Op1Class,
                                                  DAGOperand Op2Class,
-                                                 string Constraint,
+                                                 string Constraint = "",
                                                  int TargetConstraintType = 1> :
       Pseudo<(outs RetClass:$rd),
              (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2,
@@ -2056,9 +2043,10 @@ multiclass VPseudoVSFS_M {
   foreach mti = AllMasks in {
     defvar mx = mti.LMul.MX;
     let VLMul = mti.LMul.value in {
-      def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>,
+      def "_M_" # mti.BX : VPseudoUnaryNoMaskNoPolicy<VR, VR, constraint>,
                            SchedUnary<"WriteVMSFSV", "ReadVMSFSV", mx,
                                       forceMergeOpRead=true>;
+      let ForceTailAgnostic = true in
       def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>,
                                      SchedUnary<"WriteVMSFSV", "ReadVMSFSV", mx,
                                                 forceMergeOpRead=true>;
@@ -2131,8 +2119,8 @@ multiclass VPseudoBinary<VReg RetClass,
                          bit Commutable = 0> {
   let VLMul = MInfo.value, SEW=sew, isCommutable = Commutable in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
-    def suffix : VPseudoBinaryNoMaskTU<RetClass, Op1Class, Op2Class,
-                                       Constraint, TargetConstraintType>;
+    def suffix : VPseudoBinaryNoMaskPolicy<RetClass, Op1Class, Op2Class,
+                                           Constraint, TargetConstraintType>;
     def suffix # "_MASK" : VPseudoBinaryMaskPolicy<RetClass, Op1Class, Op2Class,
                                                    Constraint, TargetConstraintType>,
                            RISCVMaskedPseudo<MaskIdx=3>;
@@ -2172,8 +2160,8 @@ multiclass VPseudoBinaryM<VReg RetClass,
                           int TargetConstraintType = 1,
                           bit Commutable = 0> {
   let VLMul = MInfo.value, isCommutable = Commutable in {
-    def "_" # MInfo.MX : VPseudoBinaryMOutNoMask<RetClass, Op1Class, Op2Class,
-                                                 Constraint, TargetConstraintType>;
+    def "_" # MInfo.MX : VPseudoBinaryNoMask<RetClass, Op1Class, Op2Class,
+                                             Constraint, TargetConstraintType>;
     let ForceTailAgnostic = true in
     def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMOutMask<RetClass, Op1Class,
                                                          Op2Class, Constraint, TargetConstraintType>,
@@ -2187,11 +2175,11 @@ multiclass VPseudoBinaryEmul<VReg RetClass,
                              LMULInfo lmul,
                              LMULInfo emul,
                              string Constraint = "",
-                             int sew = 0> {
+                             int sew> {
   let VLMul = lmul.value, SEW=sew in {
     defvar suffix = !if(sew, "_" # lmul.MX # "_E" # sew, "_" # lmul.MX);
-    def suffix # "_" # emul.MX : VPseudoBinaryNoMaskTU<RetClass, Op1Class, Op2Class,
-                                                       Constraint>;
+    def suffix # "_" # emul.MX : VPseudoBinaryNoMaskPolicy<RetClass, Op1Class, Op2Class,
+                                                           Constraint>;
     def suffix # "_" # emul.MX # "_MASK" : VPseudoBinaryMaskPolicy<RetClass, Op1Class, Op2Class,
                                                                           Constraint>,
                                                   RISCVMaskedPseudo<MaskIdx=3>;
@@ -2238,18 +2226,13 @@ multiclass VPseudoBinaryV_VV_RM<LMULInfo m, string Constraint = "", bit Commutab
                                        Commutable=Commutable>;
 }
 
-// Similar to VPseudoBinaryV_VV, but uses MxListF.
-multiclass VPseudoBinaryFV_VV<LMULInfo m, string Constraint = "", int sew = 0> {
-  defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint, sew>;
-}
-
-multiclass VPseudoBinaryFV_VV_RM<LMULInfo m, string Constraint = "", int sew = 0> {
+multiclass VPseudoBinaryFV_VV_RM<LMULInfo m, int sew> {
   defm _VV : VPseudoBinaryRoundingMode<m.vrclass, m.vrclass, m.vrclass, m,
-                                       Constraint, sew,
-                                       UsesVXRM=0>;
+                                       "", sew, UsesVXRM=0>;
 }
 
-multiclass VPseudoVGTR_EI16_VV<string Constraint = ""> {
+multiclass VPseudoVGTR_EI16_VV {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
     foreach sew = EEWList in {
@@ -2263,7 +2246,7 @@ multiclass VPseudoVGTR_EI16_VV<string Constraint = ""> {
         foreach e = sews in {
           defm _VV
               : VPseudoBinaryEmul<m.vrclass, m.vrclass, emul.vrclass, m, emul,
-                                  Constraint, e>,
+                                  constraint, e>,
                 SchedBinary<"WriteVRGatherEI16VV", "ReadVRGatherEI16VV_data",
                             "ReadVRGatherEI16VV_index", mx, e, forceMergeOpRead=true>;
         }
@@ -2288,14 +2271,14 @@ multiclass VPseudoVSLD1_VX<string Constraint = ""> {
   }
 }
 
-multiclass VPseudoBinaryV_VF<LMULInfo m, FPR_Info f, string Constraint = "", int sew = 0> {
+multiclass VPseudoBinaryV_VF<LMULInfo m, FPR_Info f, int sew> {
   defm "_V" # f.FX : VPseudoBinary<m.vrclass, m.vrclass,
-                                   f.fprclass, m, Constraint, sew>;
+                                   f.fprclass, m, "", sew>;
 }
 
-multiclass VPseudoBinaryV_VF_RM<LMULInfo m, FPR_Info f, string Constraint = "", int sew = 0> {
+multiclass VPseudoBinaryV_VF_RM<LMULInfo m, FPR_Info f, int sew> {
   defm "_V" # f.FX : VPseudoBinaryRoundingMode<m.vrclass, m.vrclass,
-                                               f.fprclass, m, Constraint, sew,
+                                               f.fprclass, m, "", sew,
                                                UsesVXRM=0>;
 }
 
@@ -2310,11 +2293,11 @@ multiclass VPseudoVSLD1_VF<string Constraint = ""> {
   }
 }
 
-multiclass VPseudoBinaryV_VI<Operand ImmType = simm5, LMULInfo m, string Constraint = ""> {
+multiclass VPseudoBinaryV_VI<Operand ImmType, LMULInfo m, string Constraint = ""> {
   defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
-multiclass VPseudoBinaryV_VI_RM<Operand ImmType = simm5, LMULInfo m, string Constraint = ""> {
+multiclass VPseudoBinaryV_VI_RM<Operand ImmType, LMULInfo m, string Constraint = ""> {
   defm _VI : VPseudoBinaryRoundingMode<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
@@ -2341,7 +2324,7 @@ multiclass VPseudoBinaryW_VV<LMULInfo m, bit Commutable = 0> {
                            Commutable=Commutable>;
 }
 
-multiclass VPseudoBinaryW_VV_RM<LMULInfo m, int sew = 0> {
+multiclass VPseudoBinaryW_VV_RM<LMULInfo m, int sew> {
   defm _VV : VPseudoBinaryRoundingMode<m.wvrclass, m.vrclass, m.vrclass, m,
                                       "@earlyclobber $rd", sew, UsesVXRM=0,
                                       TargetConstraintType=3>;
@@ -2357,13 +2340,7 @@ multiclass VPseudoBinaryW_VI<Operand ImmType, LMULInfo m> {
                              "@earlyclobber $rd", TargetConstraintType=3>;
 }
 
-multiclass VPseudoBinaryW_VF<LMULInfo m, FPR_Info f> {
-  defm "_V" # f.FX : VPseudoBinary<m.wvrclass, m.vrclass,
-                                   f.fprclass, m,
-                                   "@earlyclobber $rd">;
-}
-
-multiclass VPseudoBinaryW_VF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
+multiclass VPseudoBinaryW_VF_RM<LMULInfo m, FPR_Info f, int sew> {
   defm "_V" # f.FX : VPseudoBinaryRoundingMode<m.wvrclass, m.vrclass,
                                                f.fprclass, m,
                                                "@earlyclobber $rd", sew,
@@ -2378,7 +2355,7 @@ multiclass VPseudoBinaryW_WV<LMULInfo m> {
                                "@earlyclobber $rd", TargetConstraintType=3>;
 }
 
-multiclass VPseudoBinaryW_WV_RM<LMULInfo m, int sew = 0> {
+multiclass VPseudoBinaryW_WV_RM<LMULInfo m, int sew> {
   defm _WV : VPseudoBinaryRoundingMode<m.wvrclass, m.wvrclass, m.vrclass, m,
                                        "@earlyclobber $rd", sew, UsesVXRM = 0,
                                        TargetConstraintType = 3>;
@@ -2391,12 +2368,7 @@ multiclass VPseudoBinaryW_WX<LMULInfo m> {
   defm "_WX" : VPseudoBinary<m.wvrclass, m.wvrclass, GPR, m, /*Constraint*/ "", TargetConstraintType=3>;
 }
 
-multiclass VPseudoBinaryW_WF<LMULInfo m, FPR_Info f, int TargetConstraintType = 1> {
-  defm "_W" # f.FX : VPseudoBinary<m.wvrclass, m.wvrclass,
-                                   f.fprclass, m, /*Constraint*/ "", TargetConstraintType=TargetConstraintType>;
-}
-
-multiclass VPseudoBinaryW_WF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
+multiclass VPseudoBinaryW_WF_RM<LMULInfo m, FPR_Info f, int sew> {
   defm "_W" # f.FX : VPseudoBinaryRoundingMode<m.wvrclass, m.wvrclass,
                                                f.fprclass, m,
                                                Constraint="",
@@ -2410,37 +2382,43 @@ multiclass VPseudoBinaryW_WF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
 // exception from the spec.
 // "The destination EEW is smaller than the source EEW and the overlap is in the
 //  lowest-numbered part of the source register group."
-multiclass VPseudoBinaryV_WV<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryV_WV<LMULInfo m> {
   defm _WV : VPseudoBinary<m.vrclass, m.wvrclass, m.vrclass, m,
-                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""), TargetConstraintType=TargetConstraintType>;
+                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""),
+                           TargetConstraintType=2>;
 }
 
 multiclass VPseudoBinaryV_WV_RM<LMULInfo m> {
   defm _WV : VPseudoBinaryRoundingMode<m.vrclass, m.wvrclass, m.vrclass, m,
                                        !if(!ge(m.octuple, 8),
-                                       "@earlyclobber $rd", "")>;
+                                       "@earlyclobber $rd", ""),
+                                       TargetConstraintType=2>;
 }
 
-multiclass VPseudoBinaryV_WX<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryV_WX<LMULInfo m> {
   defm _WX : VPseudoBinary<m.vrclass, m.wvrclass, GPR, m,
-                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""), TargetConstraintType=TargetConstraintType>;
+                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""),
+                           TargetConstraintType=2>;
 }
 
 multiclass VPseudoBinaryV_WX_RM<LMULInfo m> {
   defm _WX : VPseudoBinaryRoundingMode<m.vrclass, m.wvrclass, GPR, m,
                                        !if(!ge(m.octuple, 8),
-                                       "@earlyclobber $rd", "")>;
+                                       "@earlyclobber $rd", ""),
+                                       TargetConstraintType=2>;
 }
 
-multiclass VPseudoBinaryV_WI<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryV_WI<LMULInfo m> {
   defm _WI : VPseudoBinary<m.vrclass, m.wvrclass, uimm5, m,
-                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""), TargetConstraintType=TargetConstraintType>;
+                           !if(!ge(m.octuple, 8), "@earlyclobber $rd", ""),
+                           TargetConstraintType=2>;
 }
 
 multiclass VPseudoBinaryV_WI_RM<LMULInfo m> {
   defm _WI : VPseudoBinaryRoundingMode<m.vrclass, m.wvrclass, uimm5, m,
                                        !if(!ge(m.octuple, 8),
-                                       "@earlyclobber $rd", "")>;
+                                       "@earlyclobber $rd", ""),
+                                       TargetConstraintType=2>;
 }
 
 // For vadc and vsbc, the instruction encoding is reserved if the destination
@@ -2452,35 +2430,32 @@ multiclass VPseudoBinaryV_VM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
                              int TargetConstraintType = 1> {
   let isCommutable = Commutable in
   def "_VV" # !if(CarryIn, "M", "") # "_" # m.MX :
-    VPseudoBinaryCarryIn<!if(CarryOut, VR,
-                         !if(!and(CarryIn, !not(CarryOut)),
-                             GetVRegNoV0<m.vrclass>.R, m.vrclass)),
-                         m.vrclass, m.vrclass, m, CarryIn, Constraint, TargetConstraintType>;
+    VPseudoBinaryCarry<!if(CarryOut, VR,
+                       !if(!and(CarryIn, !not(CarryOut)),
+                           GetVRegNoV0<m.vrclass>.R, m.vrclass)),
+                       m.vrclass, m.vrclass, m, CarryIn, Constraint, TargetConstraintType>;
 }
 
-multiclass VPseudoTiedBinaryV_VM<LMULInfo m, int TargetConstraintType = 1,
-                                 bit Commutable = 0> {
+multiclass VPseudoTiedBinaryV_VM<LMULInfo m, bit Commutable = 0> {
   let isCommutable = Commutable in
   def "_VVM" # "_" # m.MX:
     VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                             m.vrclass, m.vrclass, m, 1, "",
-                             TargetConstraintType>;
+                             m.vrclass, m.vrclass, m>;
 }
 
 multiclass VPseudoBinaryV_XM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
                              string Constraint = "", int TargetConstraintType = 1> {
   def "_VX" # !if(CarryIn, "M", "") # "_" # m.MX :
-    VPseudoBinaryCarryIn<!if(CarryOut, VR,
-                         !if(!and(CarryIn, !not(CarryOut)),
-                             GetVRegNoV0<m.vrclass>.R, m.vrclass)),
-                         m.vrclass, GPR, m, CarryIn, Constraint, TargetConstraintType>;
+    VPseudoBinaryCarry<!if(CarryOut, VR,
+                       !if(!and(CarryIn, !not(CarryOut)),
+                           GetVRegNoV0<m.vrclass>.R, m.vrclass)),
+                       m.vrclass, GPR, m, CarryIn, Constraint, TargetConstraintType>;
 }
 
-multiclass VPseudoTiedBinaryV_XM<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoTiedBinaryV_XM<LMULInfo m> {
   def "_VXM" # "_" # m.MX:
     VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                             m.vrclass, GPR, m, 1, "",
-                             TargetConstraintType>;
+                             m.vrclass, GPR, m>;
 }
 
 multiclass VPseudoVMRG_FM {
@@ -2489,8 +2464,7 @@ multiclass VPseudoVMRG_FM {
       defvar mx = m.MX;
       def "_V" # f.FX # "M_" # mx
           : VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R, m.vrclass,
-                                     f.fprclass, m, CarryIn=1,
-                                     Constraint = "">,
+                                     f.fprclass, m>,
           SchedBinary<"WriteVFMergeV", "ReadVFMergeV", "ReadVFMergeF", mx,
                       forceMasked=1, forceMergeOpRead=true>;
     }
@@ -2500,16 +2474,16 @@ multiclass VPseudoVMRG_FM {
 multiclass VPseudoBinaryV_IM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
                              string Constraint = "", int TargetConstraintType = 1> {
   def "_VI" # !if(CarryIn, "M", "") # "_" # m.MX :
-    VPseudoBinaryCarryIn<!if(CarryOut, VR,
-                         !if(!and(CarryIn, !not(CarryOut)),
-                             GetVRegNoV0<m.vrclass>.R, m.vrclass)),
-                         m.vrclass, simm5, m, CarryIn, Constraint, TargetConstraintType>;
+    VPseudoBinaryCarry<!if(CarryOut, VR,
+                       !if(!and(CarryIn, !not(CarryOut)),
+                           GetVRegNoV0<m.vrclass>.R, m.vrclass)),
+                       m.vrclass, simm5, m, CarryIn, Constraint, TargetConstraintType>;
 }
 
 multiclass VPseudoTiedBinaryV_IM<LMULInfo m> {
   def "_VIM" # "_" # m.MX:
     VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                             m.vrclass, simm5, m, 1, "">;
+                             m.vrclass, simm5, m>;
 }
 
 multiclass VPseudoUnaryVMV_V_X_I {
@@ -2698,80 +2672,79 @@ multiclass VPseudoBinaryM_VI<LMULInfo m, int TargetConstraintType = 1> {
                             !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
 }
 
-multiclass VPseudoVGTR_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+multiclass VPseudoVGTR_VV_VX_VI {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VX<m, Constraint>,
+    defm "" : VPseudoBinaryV_VX<m, constraint>,
               SchedBinary<"WriteVRGatherVX", "ReadVRGatherVX_data",
                           "ReadVRGatherVX_index", mx, forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m, Constraint>,
+    defm "" : VPseudoBinaryV_VI<uimm5, m, constraint>,
               SchedUnary<"WriteVRGatherVI", "ReadVRGatherVI_data", mx,
                          forceMergeOpRead=true>;
 
     defvar sews = SchedSEWSet<mx>.val;
     foreach e = sews in {
-      defm "" : VPseudoBinaryV_VV<m, Constraint, e>,
+      defm "" : VPseudoBinaryV_VV<m, constraint, e>,
                 SchedBinary<"WriteVRGatherVV", "ReadVRGatherVV_data",
                               "ReadVRGatherVV_index", mx, e, forceMergeOpRead=true>;
     }
   }
 }
 
-multiclass VPseudoVSALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = "",
-                                 bit Commutable = 0> {
+multiclass VPseudoVSALU_VV_VX_VI<bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m, Constraint, Commutable=Commutable>,
+    defm "" : VPseudoBinaryV_VV<m, Commutable=Commutable>,
               SchedBinary<"WriteVSALUV", "ReadVSALUV", "ReadVSALUX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VX<m, Constraint>,
+    defm "" : VPseudoBinaryV_VX<m>,
               SchedBinary<"WriteVSALUX", "ReadVSALUV", "ReadVSALUX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m, Constraint>,
+    defm "" : VPseudoBinaryV_VI<simm5, m>,
               SchedUnary<"WriteVSALUI", "ReadVSALUV", mx, forceMergeOpRead=true>;
   }
 }
 
 
-multiclass VPseudoVSHT_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+multiclass VPseudoVSHT_VV_VX_VI {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m, Constraint>,
+    defm "" : VPseudoBinaryV_VV<m>,
               SchedBinary<"WriteVShiftV", "ReadVShiftV", "ReadVShiftV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VX<m, Constraint>,
+    defm "" : VPseudoBinaryV_VX<m>,
               SchedBinary<"WriteVShiftX", "ReadVShiftV", "ReadVShiftX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m, Constraint>,
+    defm "" : VPseudoBinaryV_VI<uimm5, m>,
               SchedUnary<"WriteVShiftI", "ReadVShiftV", mx, forceMergeOpRead=true>;
   }
 }
 
-multiclass VPseudoVSSHT_VV_VX_VI_RM<Operand ImmType = simm5, string Constraint = ""> {
+multiclass VPseudoVSSHT_VV_VX_VI_RM {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV_RM<m, Constraint>,
+    defm "" : VPseudoBinaryV_VV_RM<m>,
               SchedBinary<"WriteVSShiftV", "ReadVSShiftV", "ReadVSShiftV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VX_RM<m, Constraint>,
+    defm "" : VPseudoBinaryV_VX_RM<m>,
               SchedBinary<"WriteVSShiftX", "ReadVSShiftV", "ReadVSShiftX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI_RM<ImmType, m, Constraint>,
+    defm "" : VPseudoBinaryV_VI_RM<uimm5, m>,
               SchedUnary<"WriteVSShiftI", "ReadVSShiftV", mx, forceMergeOpRead=true>;
   }
 }
 
-multiclass VPseudoVALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = "",
-                                bit Commutable = 0> {
+multiclass VPseudoVALU_VV_VX_VI<bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m, Constraint, Commutable=Commutable>,
+    defm "" : VPseudoBinaryV_VV<m, Commutable=Commutable>,
             SchedBinary<"WriteVIALUV", "ReadVIALUV", "ReadVIALUV", mx,
                         forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VX<m, Constraint>,
+    defm "" : VPseudoBinaryV_VX<m>,
             SchedBinary<"WriteVIALUX", "ReadVIALUV", "ReadVIALUX", mx,
                         forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m, Constraint>,
+    defm "" : VPseudoBinaryV_VI<simm5, m>,
             SchedUnary<"WriteVIALUI", "ReadVIALUV", mx, forceMergeOpRead=true>;
   }
 }
@@ -2848,14 +2821,14 @@ multiclass VPseudoVDIV_VV_VX {
 multiclass VPseudoVFMUL_VV_VF_RM {
   foreach m = MxListF in {
     foreach e = SchedSEWSet<m.MX, isF=1>.val in
-      defm "" : VPseudoBinaryFV_VV_RM<m, "", sew=e>,
+      defm "" : VPseudoBinaryFV_VV_RM<m, e>,
                 SchedBinary<"WriteVFMulV", "ReadVFMulV", "ReadVFMulV", m.MX, e,
                             forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, f.SEW>,
                 SchedBinary<"WriteVFMulF", "ReadVFMulV", "ReadVFMulF", m.MX,
                             f.SEW, forceMergeOpRead=true>;
     }
@@ -2867,7 +2840,7 @@ multiclass VPseudoVFDIV_VV_VF_RM {
     defvar mx = m.MX;
     defvar sews = SchedSEWSet<mx, isF=1>.val;
     foreach e = sews in {
-      defm "" : VPseudoBinaryFV_VV_RM<m, "", e>,
+      defm "" : VPseudoBinaryFV_VV_RM<m, e>,
                 SchedBinary<"WriteVFDivV", "ReadVFDivV", "ReadVFDivV", mx, e,
                             forceMergeOpRead=true>;
     }
@@ -2875,7 +2848,7 @@ multiclass VPseudoVFDIV_VV_VF_RM {
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f, "", f.SEW>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, f.SEW>,
                 SchedBinary<"WriteVFDivF", "ReadVFDivV", "ReadVFDivF", m.MX, f.SEW,
                             forceMergeOpRead=true>;
     }
@@ -2885,7 +2858,7 @@ multiclass VPseudoVFDIV_VV_VF_RM {
 multiclass VPseudoVFRDIV_VF_RM {
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f, "", f.SEW>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, f.SEW>,
                 SchedBinary<"WriteVFDivF", "ReadVFDivV", "ReadVFDivF", m.MX, f.SEW,
                             forceMergeOpRead=true>;
     }
@@ -2906,7 +2879,7 @@ multiclass VPseudoVALU_VV_VX {
 multiclass VPseudoVSGNJ_VV_VF {
   foreach m = MxListF in {
     foreach e = SchedSEWSet<m.MX, isF=1>.val in
-    defm "" : VPseudoBinaryFV_VV<m, sew=e>,
+    defm "" : VPseudoBinaryV_VV<m, sew=e>,
               SchedBinary<"WriteVFSgnjV", "ReadVFSgnjV", "ReadVFSgnjV", m.MX,
                           e, forceMergeOpRead=true>;
   }
@@ -2923,7 +2896,7 @@ multiclass VPseudoVSGNJ_VV_VF {
 multiclass VPseudoVMAX_VV_VF {
   foreach m = MxListF in {
     foreach e = SchedSEWSet<m.MX, isF=1>.val in
-      defm "" : VPseudoBinaryFV_VV<m, sew=e>,
+      defm "" : VPseudoBinaryV_VV<m, sew=e>,
                 SchedBinary<"WriteVFMinMaxV", "ReadVFMinMaxV", "ReadVFMinMaxV",
                             m.MX, e, forceMergeOpRead=true>;
   }
@@ -2940,14 +2913,14 @@ multiclass VPseudoVMAX_VV_VF {
 multiclass VPseudoVALU_VV_VF_RM {
   foreach m = MxListF in {
     foreach e = SchedSEWSet<m.MX, isF=1>.val in
-      defm "" : VPseudoBinaryFV_VV_RM<m, "", sew=e>,
+      defm "" : VPseudoBinaryFV_VV_RM<m, e>,
                 SchedBinary<"WriteVFALUV", "ReadVFALUV", "ReadVFALUV", m.MX, e,
                             forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, f.SEW>,
                 SchedBinary<"WriteVFALUF", "ReadVFALUV", "ReadVFALUF", m.MX,
                             f.SEW, forceMergeOpRead=true>;
     }
@@ -2957,20 +2930,20 @@ multiclass VPseudoVALU_VV_VF_RM {
 multiclass VPseudoVALU_VF_RM {
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, f.SEW>,
                 SchedBinary<"WriteVFALUF", "ReadVFALUV", "ReadVFALUF", m.MX,
                             f.SEW, forceMergeOpRead=true>;
     }
   }
 }
 
-multiclass VPseudoVALU_VX_VI<Operand ImmType = simm5> {
+multiclass VPseudoVALU_VX_VI {
   foreach m = MxList in {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryV_VX<m>,
               SchedBinary<"WriteVIALUX", "ReadVIALUV", "ReadVIALUX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m>,
+    defm "" : VPseudoBinaryV_VI<simm5, m>,
               SchedUnary<"WriteVIALUI", "ReadVIALUV", mx, forceMergeOpRead=true>;
   }
 }
@@ -3066,17 +3039,17 @@ multiclass VPseudoVMRG_VM_XM_IM {
     defvar mx = m.MX;
     def "_VVM" # "_" # m.MX:
       VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                               m.vrclass, m.vrclass, m, 1, "">,
+                               m.vrclass, m.vrclass, m>,
       SchedBinary<"WriteVIMergeV", "ReadVIMergeV", "ReadVIMergeV", mx,
                           forceMergeOpRead=true>;
     def "_VXM" # "_" # m.MX:
       VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                               m.vrclass, GPR, m, 1, "">,
+                               m.vrclass, GPR, m>,
       SchedBinary<"WriteVIMergeX", "ReadVIMergeV", "ReadVIMergeX", mx,
                           forceMergeOpRead=true>;
     def "_VIM" # "_" # m.MX:
       VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                               m.vrclass, simm5, m, 1, "">,
+                               m.vrclass, simm5, m>,
       SchedUnary<"WriteVIMergeI", "ReadVIMergeV", mx,
                           forceMergeOpRead=true>;
   }
@@ -3109,57 +3082,63 @@ multiclass VPseudoVCALU_VM_XM {
   }
 }
 
-multiclass VPseudoVCALUM_VM_XM_IM<string Constraint> {
+multiclass VPseudoVCALUM_VM_XM_IM {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=Constraint,
+    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
                                 Commutable=1, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=constraint, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=1, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=1, Constraint=constraint, TargetConstraintType=2>,
               SchedUnary<"WriteVICALUI", "ReadVICALUV", mx, forceMasked=1,
                           forceMergeOpRead=true>;
   }
 }
 
-multiclass VPseudoVCALUM_VM_XM<string Constraint> {
+multiclass VPseudoVCALUM_VM_XM {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
+                                TargetConstraintType=2>,
               SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx, forceMasked=1,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=1, Constraint=constraint,
+                                TargetConstraintType=2>,
               SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx, forceMasked=1,
                           forceMergeOpRead=true>;
   }
 }
 
-multiclass VPseudoVCALUM_V_X_I<string Constraint> {
+multiclass VPseudoVCALUM_V_X_I {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=Constraint,
+    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=constraint,
                                 Commutable=1, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=0, Constraint=Constraint>,
+    defm "" : VPseudoBinaryV_IM<m, CarryOut=1, CarryIn=0, Constraint=constraint>,
               SchedUnary<"WriteVICALUI", "ReadVICALUV", mx,
                           forceMergeOpRead=true>;
   }
 }
 
-multiclass VPseudoVCALUM_V_X<string Constraint> {
+multiclass VPseudoVCALUM_V_X {
+  defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_VM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=Constraint, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_XM<m, CarryOut=1, CarryIn=0, Constraint=constraint, TargetConstraintType=2>,
               SchedBinary<"WriteVICALUX", "ReadVICALUV", "ReadVICALUX", mx,
                           forceMergeOpRead=true>;
   }
@@ -3183,13 +3162,13 @@ multiclass VPseudoVNCLP_WV_WX_WI_RM {
 multiclass VPseudoVNSHT_WV_WX_WI {
   foreach m = MxListW in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_WV<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_WV<m>,
               SchedBinary<"WriteVNShiftV", "ReadVNShiftV", "ReadVNShiftV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_WX<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_WX<m>,
               SchedBinary<"WriteVNShiftX", "ReadVNShiftV", "ReadVNShiftX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_WI<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryV_WI<m>,
               SchedUnary<"WriteVNShiftI", "ReadVNShiftV", mx,
                           forceMergeOpRead=true>;
   }
@@ -3199,34 +3178,28 @@ multiclass VPseudoTernaryWithTailPolicy<VReg RetClass,
                                           RegisterClass Op1Class,
                                           DAGOperand Op2Class,
                                           LMULInfo MInfo,
-                                          int sew,
-                                          string Constraint = "",
-                                          bit Commutable = 0> {
+                                          int sew> {
   let VLMul = MInfo.value, SEW=sew in {
     defvar mx = MInfo.MX;
-    let isCommutable = Commutable in
-    def "_" # mx # "_E" # sew : VPseudoTernaryNoMaskWithPolicy<RetClass, Op1Class, Op2Class, Constraint>;
-    def "_" # mx # "_E" # sew # "_MASK" : VPseudoTernaryMaskPolicy<RetClass, Op1Class, Op2Class, Constraint>,
+    def "_" # mx # "_E" # sew : VPseudoTernaryNoMaskWithPolicy<RetClass, Op1Class, Op2Class>;
+    def "_" # mx # "_E" # sew # "_MASK" : VPseudoTernaryMaskPolicy<RetClass, Op1Class, Op2Class>,
                                           RISCVMaskedPseudo<MaskIdx=3, MaskAffectsRes=true>;
   }
 }
 
 multiclass VPseudoTernaryWithTailPolicyRoundingMode<VReg RetClass,
-                                          RegisterClass Op1Class,
-                                          DAGOperand Op2Class,
-                                          LMULInfo MInfo,
-                                          int sew,
-                                          string Constraint = "",
-                                          bit Commutable = 0> {
+                                                    RegisterClass Op1Class,
+                                                    DAGOperand Op2Class,
+                                                    LMULInfo MInfo,
+                                                    int sew> {
   let VLMul = MInfo.value, SEW=sew in {
     defvar mx = MInfo.MX;
-    let isCommutable = Commutable in
     def "_" # mx # "_E" # sew
         : VPseudoTernaryNoMaskWithPolicyRoundingMode<RetClass, Op1Class,
-                                                     Op2Class, Constraint>;
+                                                     Op2Class>;
     def "_" # mx # "_E" # sew # "_MASK"
         : VPseudoTernaryMaskPolicyRoundingMode<RetClass, Op1Class,
-                                               Op2Class, Constraint>,
+                                               Op2Class>,
           RISCVMaskedPseudo<MaskIdx=3, MaskAffectsRes=true>;
   }
 }
@@ -3270,32 +3243,26 @@ multiclass VPseudoTernaryWithPolicyRoundingMode<VReg RetClass,
   }
 }
 
-multiclass VPseudoTernaryV_VV_AAXA<LMULInfo m, string Constraint = ""> {
+multiclass VPseudoTernaryV_VV_AAXA<LMULInfo m> {
   defm _VV : VPseudoTernaryWithPolicy<m.vrclass, m.vrclass, m.vrclass, m,
-                                      Constraint, Commutable=1>;
+                                      Commutable=1>;
 }
 
-multiclass VPseudoTernaryV_VV_AAXA_RM<LMULInfo m, string Constraint = "", int sew = 0> {
+multiclass VPseudoTernaryV_VV_AAXA_RM<LMULInfo m, int sew> {
   defm _VV : VPseudoTernaryWithPolicyRoundingMode<m.vrclass, m.vrclass, m.vrclass, m,
-                                                  Constraint, sew, Commutable=1>;
+                                                  sew=sew, Commutable=1>;
 }
 
-multiclass VPseudoTernaryV_VX_AAXA<LMULInfo m, string Constraint = ""> {
+multiclass VPseudoTernaryV_VX_AAXA<LMULInfo m> {
   defm "_VX" : VPseudoTernaryWithPolicy<m.vrclass, GPR, m.vrclass, m,
-                                        Constraint, Commutable=1>;
-}
-
-multiclass VPseudoTernaryV_VF_AAXA<LMULInfo m, FPR_Info f, string Constraint = ""> {
-  defm "_V" # f.FX : VPseudoTernaryWithPolicy<m.vrclass, f.fprclass,
-                                              m.vrclass, m, Constraint,
-                                              Commutable=1>;
+                                        Commutable=1>;
 }
 
 multiclass VPseudoTernaryV_VF_AAXA_RM<LMULInfo m, FPR_Info f,
-                                      string Constraint = "", int sew = 0> {
+                                      int sew> {
   defm "_V" # f.FX : VPseudoTernaryWithPolicyRoundingMode<m.vrclass, f.fprclass,
-                                                          m.vrclass, m, Constraint,
-                                                          sew, Commutable=1>;
+                                                          m.vrclass, m,
+                                                          sew=sew, Commutable=1>;
 }
 
 multiclass VPseudoTernaryW_VV<LMULInfo m, bit Commutable = 0> {
@@ -3304,30 +3271,24 @@ multiclass VPseudoTernaryW_VV<LMULInfo m, bit Commutable = 0> {
                                       constraint, Commutable=Commutable, TargetConstraintType=3>;
 }
 
-multiclass VPseudoTernaryW_VV_RM<LMULInfo m, int sew = 0> {
+multiclass VPseudoTernaryW_VV_RM<LMULInfo m, int sew> {
   defvar constraint = "@earlyclobber $rd";
   defm _VV : VPseudoTernaryWithPolicyRoundingMode<m.wvrclass, m.vrclass, m.vrclass, m,
-                                                  constraint, sew, /* Commutable */ 0,
+                                                  constraint, sew,
                                                   TargetConstraintType=3>;
 }
 
 multiclass VPseudoTernaryW_VX<LMULInfo m> {
   defvar constraint = "@earlyclobber $rd";
   defm "_VX" : VPseudoTernaryWithPolicy<m.wvrclass, GPR, m.vrclass, m,
-                                        constraint, /*Commutable*/ 0, TargetConstraintType=3>;
-}
-
-multiclass VPseudoTernaryW_VF<LMULInfo m, FPR_Info f, int TargetConstraintType = 1> {
-  defvar constraint = "@earlyclobber $rd";
-  defm "_V" # f.FX : VPseudoTernaryWithPolicy<m.wvrclass, f.fprclass,
-                                              m.vrclass, m, constraint, /*Commutable*/ 0, TargetConstraintType>;
+                                        constraint, TargetConstraintType=3>;
 }
 
-multiclass VPseudoTernaryW_VF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
+multiclass VPseudoTernaryW_VF_RM<LMULInfo m, FPR_Info f, int sew> {
   defvar constraint = "@earlyclobber $rd";
   defm "_V" # f.FX : VPseudoTernaryWithPolicyRoundingMode<m.wvrclass, f.fprclass,
                                                           m.vrclass, m, constraint,
-                                                          sew, /* Commutable */ 0,
+                                                          sew=sew,
                                                           TargetConstraintType=3>;
 }
 
@@ -3347,48 +3308,47 @@ multiclass VPseudoVSLDV_VX<LMULInfo m, string Constraint = ""> {
   defm _VX : VPseudoVSLDVWithPolicy<m.vrclass, m.vrclass, GPR, m, Constraint>;
 }
 
-multiclass VPseudoVSLDV_VI<Operand ImmType = simm5, LMULInfo m, string Constraint = ""> {
-  defm _VI : VPseudoVSLDVWithPolicy<m.vrclass, m.vrclass, ImmType, m, Constraint>;
+multiclass VPseudoVSLDV_VI<LMULInfo m, string Constraint = ""> {
+  defm _VI : VPseudoVSLDVWithPolicy<m.vrclass, m.vrclass, uimm5, m, Constraint>;
 }
 
-multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> {
+multiclass VPseudoVMAC_VV_VX_AAXA {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoTernaryV_VV_AAXA<m, Constraint>,
+    defm "" : VPseudoTernaryV_VV_AAXA<m>,
               SchedTernary<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV",
                            "ReadVIMulAddV", mx>;
-    defm "" : VPseudoTernaryV_VX_AAXA<m, Constraint>,
+    defm "" : VPseudoTernaryV_VX_AAXA<m>,
               SchedTernary<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX",
                            "ReadVIMulAddV", mx>;
   }
 }
 
-multiclass VPseudoVMAC_VV_VF_AAXA_RM<string Constraint = ""> {
+multiclass VPseudoVMAC_VV_VF_AAXA_RM {
   foreach m = MxListF in {
     foreach e = SchedSEWSet<m.MX, isF=1>.val in
-      defm "" : VPseudoTernaryV_VV_AAXA_RM<m, Constraint, sew=e>,
+      defm "" : VPseudoTernaryV_VV_AAXA_RM<m, sew=e>,
                 SchedTernary<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
                              "ReadVFMulAddV", m.MX, e>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoTernaryV_VF_AAXA_RM<m, f, Constraint, sew=f.SEW>,
+      defm "" : VPseudoTernaryV_VF_AAXA_RM<m, f, sew=f.SEW>,
                 SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
                              "ReadVFMulAddV", m.MX, f.SEW>;
     }
   }
 }
 
-multiclass VPseudoVSLD_VX_VI<Operand ImmType = simm5, bit slidesUp = false,
-                             string Constraint = ""> {
+multiclass VPseudoVSLD_VX_VI<bit slidesUp = false, string Constraint = ""> {
   defvar WriteSlideX = !if(slidesUp, "WriteVSlideUpX", "WriteVSlideDownX");
   foreach m = MxList in {
     defvar mx = m.MX;
     defm "" : VPseudoVSLDV_VX<m, Constraint>,
               SchedTernary<WriteSlideX, "ReadVISlideV", "ReadVISlideV",
                            "ReadVISlideX", mx>;
-    defm "" : VPseudoVSLDV_VI<ImmType, m, Constraint>,
+    defm "" : VPseudoVSLDV_VI<m, Constraint>,
               SchedBinary<"WriteVSlideI", "ReadVISlideV", "ReadVISlideV", mx>;
   }
 }
@@ -4078,9 +4038,8 @@ class VPatMaskUnaryNoMask<string intrinsic_name,
                 (mti.Mask VR:$rs2),
                 VLOpFrag)),
                 (!cast<Instruction>(inst#"_M_"#mti.BX)
-                (mti.Mask (IMPLICIT_DEF)),
                 (mti.Mask VR:$rs2),
-                GPR:$vl, mti.Log2SEW, TA_MA)>;
+                GPR:$vl, mti.Log2SEW)>;
 
 class VPatMaskUnaryMask<string intrinsic_name,
                         string inst,
@@ -4176,15 +4135,15 @@ class VPatBinaryNoMaskTURoundingMode<string intrinsic_name,
                    GPR:$vl, sew, TU_MU)>;
 
 
-// Same as above but source operands are swapped.
-class VPatBinaryNoMaskSwapped<string intrinsic_name,
-                              string inst,
-                              ValueType result_type,
-                              ValueType op1_type,
-                              ValueType op2_type,
-                              int sew,
-                              VReg op1_reg_class,
-                              DAGOperand op2_kind> :
+// Same as VPatBinaryM but source operands are swapped.
+class VPatBinaryMSwapped<string intrinsic_name,
+                         string inst,
+                         ValueType result_type,
+                         ValueType op1_type,
+                         ValueType op2_type,
+                         int sew,
+                         VReg op1_reg_class,
+                         DAGOperand op2_kind> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
                    (op2_type op2_kind:$rs2),
                    (op1_type op1_reg_class:$rs1),
@@ -4238,16 +4197,16 @@ class VPatBinaryMaskPolicy<string intrinsic_name,
                    (op2_type op2_kind:$rs2),
                    (mask_type V0), GPR:$vl, sew, (XLenVT timm:$policy))>;
 
-class VPatBinaryMaskTARoundingMode<string intrinsic_name,
-                                   string inst,
-                                   ValueType result_type,
-                                   ValueType op1_type,
-                                   ValueType op2_type,
-                                   ValueType mask_type,
-                                   int sew,
-                                   VReg result_reg_class,
-                                   VReg op1_reg_class,
-                                   DAGOperand op2_kind> :
+class VPatBinaryMaskPolicyRoundingMode<string intrinsic_name,
+                                       string inst,
+                                       ValueType result_type,
+                                       ValueType op1_type,
+                                       ValueType op2_type,
+                                       ValueType mask_type,
+                                       int sew,
+                                       VReg result_reg_class,
+                                       VReg op1_reg_class,
+                                       DAGOperand op2_kind> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name#"_mask")
                    (result_type result_reg_class:$merge),
                    (op1_type op1_reg_class:$rs1),
@@ -4263,7 +4222,7 @@ class VPatBinaryMaskTARoundingMode<string intrinsic_name,
                    (XLenVT timm:$round),
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
-// Same as above but source operands are swapped.
+// Same as VPatBinaryMask but source operands are swapped.
 class VPatBinaryMaskSwapped<string intrinsic_name,
                             string inst,
                             ValueType result_type,
@@ -4399,28 +4358,6 @@ class VPatTiedBinaryMaskRoundingMode<string intrinsic_name,
                    (XLenVT timm:$round),
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
-class VPatTernaryNoMask<string intrinsic,
-                        string inst,
-                        string kind,
-                        ValueType result_type,
-                        ValueType op1_type,
-                        ValueType op2_type,
-                        int sew,
-                        LMULInfo vlmul,
-                        VReg result_reg_class,
-                        RegisterClass op1_reg_class,
-                        DAGOperand op2_kind> :
-  Pat<(result_type (!cast<Intrinsic>(intrinsic)
-                    (result_type result_reg_class:$rs3),
-                    (op1_type op1_reg_class:$rs1),
-                    (op2_type op2_kind:$rs2),
-                    VLOpFrag)),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX)
-                    result_reg_class:$rs3,
-                    (op1_type op1_reg_class:$rs1),
-                    op2_kind:$rs2,
-                    GPR:$vl, sew)>;
-
 class VPatTernaryNoMaskTU<string intrinsic,
                           string inst,
                           string kind,
@@ -4516,31 +4453,6 @@ class VPatTernaryNoMaskWithPolicyRoundingMode<string intrinsic,
                     (XLenVT timm:$round),
                     GPR:$vl, log2sew, (XLenVT timm:$policy))>;
 
-class VPatTernaryMask<string intrinsic,
-                      string inst,
-                      string kind,
-                      ValueType result_type,
-                      ValueType op1_type,
-                      ValueType op2_type,
-                      ValueType mask_type,
-                      int sew,
-                      LMULInfo vlmul,
-                      VReg result_reg_class,
-                      RegisterClass op1_reg_class,
-                      DAGOperand op2_kind> :
-  Pat<(result_type (!cast<Intrinsic>(intrinsic#"_mask")
-                    (result_type result_reg_class:$rs3),
-                    (op1_type op1_reg_class:$rs1),
-                    (op2_type op2_kind:$rs2),
-                    (mask_type V0),
-                    VLOpFrag)),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX # "_MASK")
-                    result_reg_class:$rs3,
-                    (op1_type op1_reg_class:$rs1),
-                    op2_kind:$rs2,
-                    (mask_type V0),
-                    GPR:$vl, sew)>;
-
 class VPatTernaryMaskPolicy<string intrinsic,
                             string inst,
                             string kind,
@@ -4808,23 +4720,23 @@ multiclass VPatBinaryRoundingMode<string intrinsic,
                                   DAGOperand op2_kind> {
   def : VPatBinaryNoMaskTURoundingMode<intrinsic, inst, result_type, op1_type, op2_type,
                                        sew, result_reg_class, op1_reg_class, op2_kind>;
-  def : VPatBinaryMaskTARoundingMode<intrinsic, inst, result_type, op1_type, op2_type,
-                                     mask_type, sew, result_reg_class, op1_reg_class,
-                                     op2_kind>;
+  def : VPatBinaryMaskPolicyRoundingMode<intrinsic, inst, result_type, op1_type, op2_type,
+                                         mask_type, sew, result_reg_class, op1_reg_class,
+                                         op2_kind>;
 }
 
-multiclass VPatBinarySwapped<string intrinsic,
-                      string inst,
-                      ValueType result_type,
-                      ValueType op1_type,
-                      ValueType op2_type,
-                      ValueType mask_type,
-                      int sew,
-                      VReg result_reg_class,
-                      VReg op1_reg_class,
-                      DAGOperand op2_kind> {
-  def : VPatBinaryNoMaskSwapped<intrinsic, inst, result_type, op1_type, op2_type,
-                                sew, op1_reg_class, op2_kind>;
+multiclass VPatBinaryMSwapped<string intrinsic,
+                              string inst,
+                              ValueType result_type,
+                              ValueType op1_type,
+                              ValueType op2_type,
+                              ValueType mask_type,
+                              int sew,
+                              VReg result_reg_class,
+                              VReg op1_reg_class,
+                              DAGOperand op2_kind> {
+  def : VPatBinaryMSwapped<intrinsic, inst, result_type, op1_type, op2_type,
+                           sew, op1_reg_class, op2_kind>;
   def : VPatBinaryMaskSwapped<intrinsic, inst, result_type, op1_type, op2_type,
                               mask_type, sew, result_reg_class, op1_reg_class,
                               op2_kind>;
@@ -5182,10 +5094,10 @@ multiclass VPatBinaryW_WV_RM<string intrinsic, string instruction,
                                            Wti.Vector, Vti.Vector, Vti.Mask,
                                            Vti.Log2SEW, Wti.RegClass, Vti.RegClass>;
       }
-      def : VPatBinaryMaskTARoundingMode<intrinsic, name,
-                                         Wti.Vector, Wti.Vector, Vti.Vector, Vti.Mask,
-                                         Vti.Log2SEW, Wti.RegClass,
-                                         Wti.RegClass, Vti.RegClass>;
+      def : VPatBinaryMaskPolicyRoundingMode<intrinsic, name,
+                                             Wti.Vector, Wti.Vector, Vti.Vector, Vti.Mask,
+                                             Vti.Log2SEW, Wti.RegClass,
+                                             Wti.RegClass, Vti.RegClass>;
     }
   }
 }
@@ -5421,10 +5333,10 @@ multiclass VPatBinarySwappedM_VV<string intrinsic, string instruction,
                                  list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
     let Predicates = GetVTypePredicates<vti>.Predicates in
-    defm : VPatBinarySwapped<intrinsic, instruction # "_VV_" # vti.LMul.MX,
-                             vti.Mask, vti.Vector, vti.Vector, vti.Mask,
-                             vti.Log2SEW, VR,
-                             vti.RegClass, vti.RegClass>;
+    defm : VPatBinaryMSwapped<intrinsic, instruction # "_VV_" # vti.LMul.MX,
+                              vti.Mask, vti.Vector, vti.Vector, vti.Mask,
+                              vti.Log2SEW, VR,
+                              vti.RegClass, vti.RegClass>;
 }
 
 multiclass VPatBinaryM_VX<string intrinsic, string instruction,
@@ -5456,7 +5368,7 @@ multiclass VPatBinaryV_VV_VX_VI<string intrinsic, string instruction,
       VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>;
 
 multiclass VPatBinaryV_VV_VX_VI_RM<string intrinsic, string instruction,
-                                   list<VTypeInfo> vtilist, Operand ImmType = simm5>
+                                   list<VTypeInfo> vtilist, Operand ImmType>
     : VPatBinaryV_VV_RM<intrinsic, instruction, vtilist>,
       VPatBinaryV_VX_RM<intrinsic, instruction, vtilist>,
       VPatBinaryV_VI_RM<intrinsic, instruction, vtilist, ImmType>;
@@ -5537,46 +5449,6 @@ multiclass VPatBinaryM_V_X<string intrinsic, string instruction>
     : VPatBinaryV_V<intrinsic, instruction>,
       VPatBinaryV_X<intrinsic, instruction>;
 
-multiclass VPatTernary<string intrinsic,
-                       string inst,
-                       string kind,
-                       ValueType result_type,
-                       ValueType op1_type,
-                       ValueType op2_type,
-                       ValueType mask_type,
-                       int sew,
-                       LMULInfo vlmul,
-                       VReg result_reg_class,
-                       RegisterClass op1_reg_class,
-                       DAGOperand op2_kind> {
-  def : VPatTernaryNoMask<intrinsic, inst, kind, result_type, op1_type, op2_type,
-                          sew, vlmul, result_reg_class, op1_reg_class,
-                          op2_kind>;
-  def : VPatTernaryMask<intrinsic, inst, kind, result_type, op1_type, op2_type,
-                        mask_type, sew, vlmul, result_reg_class, op1_reg_class,
-                        op2_kind>;
-}
-
-multiclass VPatTernaryNoMaskNoPolicy<string intrinsic,
-                                     string inst,
-                                     string kind,
-                                     ValueType result_type,
-                                     ValueType op1_type,
-                                     ValueType op2_type,
-                                     ValueType mask_type,
-                                     int sew,
-                                     LMULInfo vlmul,
-                                     VReg result_reg_class,
-                                     RegisterClass op1_reg_class,
-                                     DAGOperand op2_kind> {
-  def : VPatTernaryNoMask<intrinsic, inst, kind, result_type, op1_type, op2_type,
-                          sew, vlmul, result_reg_class, op1_reg_class,
-                          op2_kind>;
-  def : VPatTernaryMaskPolicy<intrinsic, inst, kind, result_type, op1_type, op2_type,
-                              mask_type, sew, vlmul, result_reg_class, op1_reg_class,
-                              op2_kind>;
-}
-
 multiclass VPatTernaryWithPolicy<string intrinsic,
                                  string inst,
                                  string kind,
@@ -5792,7 +5664,7 @@ multiclass VPatTernaryV_VV_VX_AAXA_RM<string intrinsic, string instruction,
       VPatTernaryV_VX_AAXA_RM<intrinsic, instruction, vtilist, isSEWAware>;
 
 multiclass VPatTernaryV_VX_VI<string intrinsic, string instruction,
-                              list<VTypeInfo> vtilist, Operand Imm_type = simm5>
+                              list<VTypeInfo> vtilist, Operand Imm_type>
     : VPatTernaryV_VX<intrinsic, instruction, vtilist>,
       VPatTernaryV_VI<intrinsic, instruction, vtilist, Imm_type>;
 
@@ -5824,7 +5696,7 @@ multiclass VPatBinaryM_VX_VI<string intrinsic, string instruction,
       VPatBinaryM_VI<intrinsic, instruction, vtilist>;
 
 multiclass VPatBinaryV_VV_VX_VI_INT<string intrinsic, string instruction,
-                                    list<VTypeInfo> vtilist, Operand ImmType = simm5>
+                                    list<VTypeInfo> vtilist, Operand ImmType>
     : VPatBinaryV_VV_INT<intrinsic#"_vv", instruction, vtilist>,
       VPatBinaryV_VX_INT<intrinsic#"_vx", instruction, vtilist>,
       VPatBinaryV_VI<intrinsic#"_vx", instruction, vtilist, ImmType>;
@@ -6327,12 +6199,12 @@ defm PseudoVSEXT_VF8 : PseudoVEXT_VF8;
 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
 //===----------------------------------------------------------------------===//
 defm PseudoVADC  : VPseudoVCALU_VM_XM_IM;
-defm PseudoVMADC : VPseudoVCALUM_VM_XM_IM<"@earlyclobber $rd">;
-defm PseudoVMADC : VPseudoVCALUM_V_X_I<"@earlyclobber $rd">;
+defm PseudoVMADC : VPseudoVCALUM_VM_XM_IM;
+defm PseudoVMADC : VPseudoVCALUM_V_X_I;
 
 defm PseudoVSBC  : VPseudoVCALU_VM_XM;
-defm PseudoVMSBC : VPseudoVCALUM_VM_XM<"@earlyclobber $rd">;
-defm PseudoVMSBC : VPseudoVCALUM_V_X<"@earlyclobber $rd">;
+defm PseudoVMSBC : VPseudoVCALUM_VM_XM;
+defm PseudoVMSBC : VPseudoVCALUM_V_X;
 
 //===----------------------------------------------------------------------===//
 // 11.5. Vector Bitwise Logical Instructions
@@ -6344,9 +6216,9 @@ defm PseudoVXOR : VPseudoVALU_VV_VX_VI<Commutable=1>;
 //===----------------------------------------------------------------------===//
 // 11.6. Vector Single-Width Bit Shift Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVSLL : VPseudoVSHT_VV_VX_VI<uimm5>;
-defm PseudoVSRL : VPseudoVSHT_VV_VX_VI<uimm5>;
-defm PseudoVSRA : VPseudoVSHT_VV_VX_VI<uimm5>;
+defm PseudoVSLL : VPseudoVSHT_VV_VX_VI;
+defm PseudoVSRL : VPseudoVSHT_VV_VX_VI;
+defm PseudoVSRA : VPseudoVSHT_VV_VX_VI;
 
 //===----------------------------------------------------------------------===//
 // 11.7. Vector Narrowing Integer Right Shift Instructions
@@ -6455,8 +6327,8 @@ let Defs = [VXSAT] in {
 //===----------------------------------------------------------------------===//
 // 12.4. Vector Single-Width Scaling Shift Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVSSRL : VPseudoVSSHT_VV_VX_VI_RM<uimm5>;
-defm PseudoVSSRA : VPseudoVSSHT_VV_VX_VI_RM<uimm5>;
+defm PseudoVSSRL : VPseudoVSSHT_VV_VX_VI_RM;
+defm PseudoVSSRA : VPseudoVSSHT_VV_VX_VI_RM;
 
 //===----------------------------------------------------------------------===//
 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
@@ -6841,8 +6713,8 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
 // 16.3. Vector Slide Instructions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasVInstructions] in {
-  defm PseudoVSLIDEUP    : VPseudoVSLD_VX_VI<uimm5, /*slidesUp=*/true, "@earlyclobber $rd">;
-  defm PseudoVSLIDEDOWN  : VPseudoVSLD_VX_VI<uimm5, /*slidesUp=*/false>;
+  defm PseudoVSLIDEUP    : VPseudoVSLD_VX_VI</*slidesUp=*/true, "@earlyclobber $rd">;
+  defm PseudoVSLIDEDOWN  : VPseudoVSLD_VX_VI</*slidesUp=*/false>;
   defm PseudoVSLIDE1UP   : VPseudoVSLD1_VX<"@earlyclobber $rd">;
   defm PseudoVSLIDE1DOWN : VPseudoVSLD1_VX;
 } // Predicates = [HasVInstructions]
@@ -6856,8 +6728,8 @@ let Predicates = [HasVInstructionsAnyF] in {
 // 16.4. Vector Register Gather Instructions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasVInstructions] in {
-defm PseudoVRGATHER     : VPseudoVGTR_VV_VX_VI<uimm5, "@earlyclobber $rd">;
-defm PseudoVRGATHEREI16 : VPseudoVGTR_EI16_VV<Constraint = "@earlyclobber $rd">;
+defm PseudoVRGATHER     : VPseudoVGTR_VV_VX_VI;
+defm PseudoVRGATHEREI16 : VPseudoVGTR_EI16_VV;
 
 //===----------------------------------------------------------------------===//
 // 16.5. Vector Compress Instruction
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 3163e4bafd4b0dcbb0ff3e81dd7b988cb064a343..497c4aadf7535fe4ec57f6997c7bff76309cfdab 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1394,7 +1394,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV<SETOLE, "PseudoVMFLE", "PseudoVMFGE">;
 // Floating-point vselects:
 // 11.15. Vector Integer Merge Instructions
 // 13.15. Vector Floating-Point Merge Instruction
-foreach fvti = AllFloatVectors in {
+foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
   defvar ivti = GetIntVTypeInfo<fvti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(fvti.Vector (vselect (fvti.Mask V0), fvti.RegClass:$rs1,
@@ -1412,7 +1412,9 @@ foreach fvti = AllFloatVectors in {
                    fvti.RegClass:$rs2, 0, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>;
 
   }
-  let Predicates = GetVTypePredicates<fvti>.Predicates in 
+
+  let Predicates = !listconcat(GetVTypePredicates<GetFpVTypeInfo<fvti>.Vti>.Predicates,
+                               GetVTypeScalarPredicates<fvti>.Predicates) in 
     def : Pat<(fvti.Vector (vselect (fvti.Mask V0),
                                     (SplatFPOp fvti.ScalarRegClass:$rs1),
                                     fvti.RegClass:$rs2)),
@@ -1475,7 +1477,7 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
 //===----------------------------------------------------------------------===//
 
 foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
-  let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
+  let Predicates = !listconcat(GetVTypePredicates<GetFpVTypeInfo<fvti>.Vti>.Predicates,
                                GetVTypeScalarPredicates<fvti>.Predicates) in
     def : Pat<(fvti.Vector (riscv_vfmv_v_f_vl undef, fvti.ScalarRegClass:$rs1, srcvalue)),
               (!cast<Instruction>("PseudoVFMV_V_"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index ce8133a5a297b922b8952c8cfebec903cc0c8f32..70d8265e7be46408fd92da58a4be0df4b0bc6dbe 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2604,7 +2604,7 @@ foreach vti = AllFloatVectors in {
   }
 }
 
-foreach fvti = AllFloatVectors in {
+foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
   // Floating-point vselects:
   // 11.15. Vector Integer Merge Instructions
   // 13.15. Vector Floating-Point Merge Instruction
@@ -2639,7 +2639,8 @@ foreach fvti = AllFloatVectors in {
                  GPR:$vl, fvti.Log2SEW)>;
   }
 
-  let Predicates = GetVTypePredicates<fvti>.Predicates in {
+  let Predicates = !listconcat(GetVTypePredicates<GetFpVTypeInfo<fvti>.Vti>.Predicates,
+                               GetVTypeScalarPredicates<fvti>.Predicates) in {
     def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0),
                                             (SplatFPOp fvti.ScalarRegClass:$rs1),
                                             fvti.RegClass:$rs2,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index 6dae8ca8f7a84ebbe853e02a7134ffbef95129d4..f0d6913a9d3fec8ad72c7f382eaa1a5f019103a7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -512,11 +512,13 @@ def CVrrAsmOperand : AsmOperandClass {
   let DiagnosticType = "InvalidRegReg";
 }
 
-def CVrr : Operand<OtherVT> {
+def CVrr : Operand<i32>,
+           ComplexPattern<i32, 2, "SelectAddrRegReg",[]> {
    let ParserMatchClass = CVrrAsmOperand;
    let EncoderMethod =  "getRegReg";
    let DecoderMethod = "decodeRegReg";
    let PrintMethod = "printRegReg";
+   let MIOperandInfo = (ops GPR:$base, GPR:$offset);
 }
 
 class CVLoad_ri_inc<bits<3> funct3, string opcodestr>
@@ -659,6 +661,47 @@ let Predicates = [HasVendorXCVelw, IsRV32], hasSideEffects = 0,
   def CV_ELW : CVLoad_ri<0b011, "cv.elw">;
 }
 
+//===----------------------------------------------------------------------===//
+// Patterns for load & store operations
+//===----------------------------------------------------------------------===//
+class CVLdrrPat<PatFrag LoadOp, RVInst Inst>
+    : Pat<(XLenVT (LoadOp CVrr:$regreg)),
+          (Inst CVrr:$regreg)>;
+
+class CVStriPat<PatFrag StoreOp, RVInst Inst>
+    : Pat<(StoreOp (XLenVT GPR:$rs2), GPR:$rs1, simm12:$imm12),
+          (Inst GPR:$rs2, GPR:$rs1, simm12:$imm12)>;
+
+class CVStrriPat<PatFrag StoreOp, RVInst Inst>
+    : Pat<(StoreOp (XLenVT GPR:$rs2), GPR:$rs1, GPR:$rs3),
+          (Inst GPR:$rs2, GPR:$rs1, GPR:$rs3)>;
+
+class CVStrrPat<PatFrag StoreOp, RVInst Inst>
+    : Pat<(StoreOp (XLenVT GPR:$rs2), CVrr:$regreg),
+          (Inst GPR:$rs2, CVrr:$regreg)>;
+
+let Predicates = [HasVendorXCVmem, IsRV32], AddedComplexity = 1 in {
+  def : CVLdrrPat<sextloadi8, CV_LB_rr>;
+  def : CVLdrrPat<zextloadi8, CV_LBU_rr>;
+  def : CVLdrrPat<extloadi8, CV_LBU_rr>;
+  def : CVLdrrPat<sextloadi16, CV_LH_rr>;
+  def : CVLdrrPat<zextloadi16, CV_LHU_rr>;
+  def : CVLdrrPat<extloadi16, CV_LHU_rr>;
+  def : CVLdrrPat<load, CV_LW_rr>;
+
+  def : CVStriPat<post_truncsti8, CV_SB_ri_inc>;
+  def : CVStriPat<post_truncsti16, CV_SH_ri_inc>;
+  def : CVStriPat<post_store, CV_SW_ri_inc>;
+
+  def : CVStrriPat<post_truncsti8, CV_SB_rr_inc>;
+  def : CVStrriPat<post_truncsti16, CV_SH_ri_inc>;
+  def : CVStrriPat<post_store, CV_SW_rr_inc>;
+
+  def : CVStrrPat<truncstorei8, CV_SB_rr>;
+  def : CVStrrPat<truncstorei16, CV_SH_rr>;
+  def : CVStrrPat<store, CV_SW_rr>;
+}
+
 def cv_tuimm2 : TImmLeaf<XLenVT, [{return isUInt<2>(Imm);}]>;
 def cv_tuimm5 : TImmLeaf<XLenVT, [{return isUInt<5>(Imm);}]>;
 def cv_uimm10 : ImmLeaf<XLenVT, [{return isUInt<10>(Imm);}]>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index d091077f729b8a7ae441e772fa24c29239c66521..75fcc1e7cb11073df3e75d5ca4d5ff8cb5643f3c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -247,23 +247,21 @@ class VPseudoTernaryNoMask_Zvk<VReg RetClass,
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
-multiclass VPseudoBinaryNoMaskTU_Zvk<VReg RetClass,
-                                     VReg Op1Class,
-                                     DAGOperand Op2Class,
-                                     LMULInfo MInfo,
-                                     string Constraint = "",
-                                     int sew = 0> {
-  let VLMul = MInfo.value, SEW=sew in {
-    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
-    def suffix : VPseudoBinaryNoMaskTU<RetClass, Op1Class, Op2Class,
-                                       Constraint>;
+multiclass VPseudoBinaryNoMaskPolicy_Zvk<VReg RetClass,
+                                         VReg Op1Class,
+                                         DAGOperand Op2Class,
+                                         LMULInfo MInfo,
+                                         string Constraint = ""> {
+  let VLMul = MInfo.value in {
+    def "_" # MInfo.MX : VPseudoBinaryNoMaskPolicy<RetClass, Op1Class, Op2Class,
+                                           Constraint>;
   }
 }
 
 multiclass VPseudoTernaryNoMask_Zvk<VReg RetClass,
-                                   VReg Op1Class,
-                                   DAGOperand Op2Class,
-                                   LMULInfo MInfo> {
+                                    VReg Op1Class,
+                                    DAGOperand Op2Class,
+                                    LMULInfo MInfo> {
   let VLMul = MInfo.value in
     def "_" # MInfo.MX : VPseudoTernaryNoMask_Zvk<RetClass, Op1Class, Op2Class>;
 }
@@ -349,7 +347,7 @@ multiclass VPseudoVSHA2MS {
 multiclass VPseudoVAESKF1 {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
-    defm _VI : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, uimm5, m>,
+    defm _VI : VPseudoBinaryNoMaskPolicy_Zvk<m.vrclass, m.vrclass, uimm5, m>,
                SchedBinary<"WriteVAESKF1V", "ReadVAESKF1V", "ReadVAESKF1V", mx,
                            forceMergeOpRead=true>;
   }
@@ -384,7 +382,7 @@ multiclass VPseudoVSM3C {
 multiclass VPseudoVSM4K {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
-    defm _VI : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, uimm5, m>,
+    defm _VI : VPseudoBinaryNoMaskPolicy_Zvk<m.vrclass, m.vrclass, uimm5, m>,
                SchedBinary<"WriteVSM4KV", "ReadVSM4KV", "ReadVSM4KV", mx,
                            forceMergeOpRead=true>;
   }
@@ -393,7 +391,7 @@ multiclass VPseudoVSM4K {
 multiclass VPseudoVSM3ME {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
-    defm _VV : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
+    defm _VV : VPseudoBinaryNoMaskPolicy_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
                SchedBinary<"WriteVSM3MEV", "ReadVSM3MEV", "ReadVSM3MEV", mx,
                            forceMergeOpRead=true>;
   }
@@ -452,16 +450,16 @@ multiclass VPseudoVCPOP {
   }
 }
 
-multiclass VPseudoVWALU_VV_VX_VI<Operand ImmType> {
+multiclass VPseudoVWSLL {
   foreach m = MxListW in {
     defvar mx = m.MX;
     defm "" : VPseudoBinaryW_VV<m>,
               SchedBinary<"WriteVWSLLV", "ReadVWSLLV", "ReadVWSLLV", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryW_VX<m>, 
+    defm "" : VPseudoBinaryW_VX<m>,
               SchedBinary<"WriteVWSLLX", "ReadVWSLLV", "ReadVWSLLX", mx,
                           forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryW_VI<ImmType, m>,
+    defm "" : VPseudoBinaryW_VI<uimm5, m>,
               SchedUnary<"WriteVWSLLI", "ReadVWSLLV", mx,
                          forceMergeOpRead=true>;
   }
@@ -494,7 +492,7 @@ multiclass VPseudoVREV8 {
   }
 }
 
-multiclass VPseudoVROL {
+multiclass VPseudoVROT_VV_VX {
  foreach m = MxList in {
     defm "" : VPseudoBinaryV_VV<m>,
               SchedBinary<"WriteVRotV", "ReadVRotV", "ReadVRotV", m.MX,
@@ -505,18 +503,12 @@ multiclass VPseudoVROL {
   }
 }
 
-multiclass VPseudoVROR<Operand ImmType> {
-  defvar Constraint = "";
+multiclass VPseudoVROT_VV_VX_VI
+   : VPseudoVROT_VV_VX {
   foreach m = MxList in {
-    defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m>,
-              SchedBinary<"WriteVRotV", "ReadVRotV", "ReadVRotV", mx,
-                          forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VX<m>,
-              SchedBinary<"WriteVRotX", "ReadVRotV", "ReadVRotX", mx,
-                          forceMergeOpRead=true>;
-    defm "" : VPseudoBinaryV_VI<ImmType, m>,
-              SchedUnary<"WriteVRotI", "ReadVRotV", mx, forceMergeOpRead=true>;
+    defm "" : VPseudoBinaryV_VI<uimm6, m>,
+              SchedUnary<"WriteVRotI", "ReadVRotV", m.MX,
+                         forceMergeOpRead=true>;
   }
 }
 
@@ -525,7 +517,7 @@ let Predicates = [HasStdExtZvbb] in {
   defm PseudoVCLZ   : VPseudoVCLZ;
   defm PseudoVCTZ   : VPseudoVCTZ;
   defm PseudoVCPOP  : VPseudoVCPOP;
-  defm PseudoVWSLL : VPseudoVWALU_VV_VX_VI<uimm5>;
+  defm PseudoVWSLL : VPseudoVWSLL;
 } // Predicates = [HasStdExtZvbb]
 
 let Predicates = [HasStdExtZvbc] in {
@@ -537,8 +529,8 @@ let Predicates = [HasStdExtZvkb] in {
   defm PseudoVANDN  : VPseudoVANDN;
   defm PseudoVBREV8 : VPseudoVBREV8;
   defm PseudoVREV8  : VPseudoVREV8;
-  defm PseudoVROL   : VPseudoVROL;
-  defm PseudoVROR   : VPseudoVROR<uimm6>;
+  defm PseudoVROL   : VPseudoVROT_VV_VX;
+  defm PseudoVROR   : VPseudoVROT_VV_VX_VI;
 } // Predicates = [HasStdExtZvkb]
 
 let Predicates = [HasStdExtZvkg] in {
@@ -1031,45 +1023,32 @@ multiclass VPatBinaryV_VV_VX_VROL<string intrinsic, string instruction,
       VPatBinaryV_VI_VROL<intrinsic, instruction2, vtilist>;
 
 multiclass VPatBinaryV_VV_VX_VI_VROR<string intrinsic, string instruction,
-                                     list<VTypeInfo> vtilist,
-                                     Operand ImmType = uimm6>
+                                     list<VTypeInfo> vtilist>
     : VPatBinaryV_VV<intrinsic, instruction, vtilist>,
       VPatBinaryV_VX_VROTATE<intrinsic, instruction, vtilist>,
-      VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>;
-
-multiclass VPatBinaryW_VI_VWSLL<string intrinsic, string instruction,
-                                list<VTypeInfoToWide> vtilist> {
-  foreach VtiToWti = vtilist in {
-    defvar Vti = VtiToWti.Vti;
-    defvar Wti = VtiToWti.Wti;
-    defm : VPatBinary<intrinsic, instruction # "_VI_" # Vti.LMul.MX,
-                      Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
-                      Vti.Log2SEW, Wti.RegClass,
-                      Vti.RegClass, uimm5>;
-  }
-}
+      VPatBinaryV_VI<intrinsic, instruction, vtilist, uimm6>;
 
-multiclass VPatBinaryW_VX_VWSLL<string intrinsic, string instruction,
-                                list<VTypeInfoToWide> vtilist> {
+multiclass VPatBinaryW_VV_VX_VI_VWSLL<string intrinsic, string instruction,
+                                      list<VTypeInfoToWide> vtilist>
+    : VPatBinaryW_VV<intrinsic, instruction, vtilist> {
   foreach VtiToWti = vtilist in {
     defvar Vti = VtiToWti.Vti;
     defvar Wti = VtiToWti.Wti;
     defvar kind = "V"#Vti.ScalarSuffix;
     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
-                                 GetVTypePredicates<Wti>.Predicates) in
-    defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX,
-                      Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
-                      Vti.Log2SEW, Wti.RegClass,
-                      Vti.RegClass, Vti.ScalarRegClass>;
+                                 GetVTypePredicates<Wti>.Predicates) in {
+      defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX,
+                        Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
+                        Vti.Log2SEW, Wti.RegClass,
+                        Vti.RegClass, Vti.ScalarRegClass>;
+      defm : VPatBinary<intrinsic, instruction # "_VI_" # Vti.LMul.MX,
+                        Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
+                        Vti.Log2SEW, Wti.RegClass,
+                        Vti.RegClass, uimm5>;
+    }
   }
 }
 
-multiclass VPatBinaryW_VV_VX_VI_VWSLL<string intrinsic, string instruction,
-                                      list<VTypeInfoToWide> vtilist>
-    : VPatBinaryW_VV<intrinsic, instruction, vtilist>,
-      VPatBinaryW_VX_VWSLL<intrinsic, instruction, vtilist>,
-      VPatBinaryW_VI_VWSLL<intrinsic, instruction, vtilist>;
-
 let Predicates = [HasStdExtZvbb] in {
   defm : VPatUnaryV_V<"int_riscv_vbrev", "PseudoVBREV", AllIntegerVectors>;
   defm : VPatUnaryV_V<"int_riscv_vclz", "PseudoVCLZ", AllIntegerVectors>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 956b851fce6c78946fdecd1011b1ab55340f0e4c..49838e685a6d2b5f979c3c11e2258c883bf01adc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1459,11 +1459,22 @@ static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call,
         Component == 3 ? NumActualRetComponents - 1 : Component;
     assert(ExtractedComposite < NumActualRetComponents &&
            "Invalid composite index!");
+    Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
+    SPIRVType *NewType = nullptr;
+    if (QueryResultType->getOpcode() == SPIRV::OpTypeVector) {
+      Register NewTypeReg = QueryResultType->getOperand(1).getReg();
+      if (TypeReg != NewTypeReg &&
+          (NewType = GR->getSPIRVTypeForVReg(NewTypeReg)) != nullptr)
+        TypeReg = NewTypeReg;
+    }
     MIRBuilder.buildInstr(SPIRV::OpCompositeExtract)
         .addDef(Call->ReturnRegister)
-        .addUse(GR->getSPIRVTypeID(Call->ReturnType))
+        .addUse(TypeReg)
         .addUse(QueryResult)
         .addImm(ExtractedComposite);
+    if (NewType != nullptr)
+      insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
+                        MIRBuilder.getMF().getRegInfo());
   } else {
     // More than 1 component is expected, fill a new vector.
     auto MIB = MIRBuilder.buildInstr(SPIRV::OpVectorShuffle)
@@ -2063,16 +2074,30 @@ static bool generateAsyncCopy(const SPIRV::IncomingCall *Call,
   auto Scope = buildConstantIntReg(SPIRV::Scope::Workgroup, MIRBuilder, GR);
 
   switch (Opcode) {
-  case SPIRV::OpGroupAsyncCopy:
-    return MIRBuilder.buildInstr(Opcode)
-        .addDef(Call->ReturnRegister)
-        .addUse(GR->getSPIRVTypeID(Call->ReturnType))
-        .addUse(Scope)
-        .addUse(Call->Arguments[0])
-        .addUse(Call->Arguments[1])
-        .addUse(Call->Arguments[2])
-        .addUse(buildConstantIntReg(1, MIRBuilder, GR))
-        .addUse(Call->Arguments[3]);
+  case SPIRV::OpGroupAsyncCopy: {
+    SPIRVType *NewType =
+        Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent
+            ? nullptr
+            : GR->getOrCreateSPIRVTypeByName("spirv.Event", MIRBuilder);
+    Register TypeReg = GR->getSPIRVTypeID(NewType ? NewType : Call->ReturnType);
+    unsigned NumArgs = Call->Arguments.size();
+    Register EventReg = Call->Arguments[NumArgs - 1];
+    bool Res = MIRBuilder.buildInstr(Opcode)
+                   .addDef(Call->ReturnRegister)
+                   .addUse(TypeReg)
+                   .addUse(Scope)
+                   .addUse(Call->Arguments[0])
+                   .addUse(Call->Arguments[1])
+                   .addUse(Call->Arguments[2])
+                   .addUse(Call->Arguments.size() > 4
+                               ? Call->Arguments[3]
+                               : buildConstantIntReg(1, MIRBuilder, GR))
+                   .addUse(EventReg);
+    if (NewType != nullptr)
+      insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
+                        MIRBuilder.getMF().getRegInfo());
+    return Res;
+  }
   case SPIRV::OpGroupWaitEvents:
     return MIRBuilder.buildInstr(Opcode)
         .addUse(Scope)
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 24c6c2688642e7ce6a29ae74fae826f406077ead..edc9e1a33d9f5a47f7aec8776b4c5b60bfb36c18 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -586,6 +586,7 @@ defm : DemangledNativeBuiltin<"__spirv_SpecConstantComposite", OpenCL_std, SpecC
 
 // Async Copy and Prefetch builtin records:
 defm : DemangledNativeBuiltin<"async_work_group_copy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>;
+defm : DemangledNativeBuiltin<"async_work_group_strided_copy", OpenCL_std, AsyncCopy, 5, 5, OpGroupAsyncCopy>;
 defm : DemangledNativeBuiltin<"__spirv_GroupAsyncCopy", OpenCL_std, AsyncCopy, 6, 6, OpGroupAsyncCopy>;
 defm : DemangledNativeBuiltin<"wait_group_events", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>;
 defm : DemangledNativeBuiltin<"__spirv_GroupWaitEvents", OpenCL_std, AsyncCopy, 3, 3, OpGroupWaitEvents>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index f4daab7d06eb539ab3036904f58d70dcebd47631..de0b97fbea0c18b83519abc2d4c86f3ebc485447 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -243,11 +243,8 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
       continue;
 
     MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1));
-    Type *ElementTy = cast<ConstantAsMetadata>(VMD->getMetadata())->getType();
-    if (isUntypedPointerTy(ElementTy))
-      ElementTy =
-          TypedPointerType::get(IntegerType::getInt8Ty(II->getContext()),
-                                getPointerAddressSpace(ElementTy));
+    Type *ElementTy =
+        toTypedPointer(cast<ConstantAsMetadata>(VMD->getMetadata())->getType());
     SPIRVType *ElementType = GR->getOrCreateSPIRVType(ElementTy, MIRBuilder);
     return GR->getOrCreateSPIRVPointerType(
         ElementType, MIRBuilder,
@@ -257,12 +254,8 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
 
   // Replace PointerType with TypedPointerType to be able to map SPIR-V types to
   // LLVM types in a consistent manner
-  if (isUntypedPointerTy(OriginalArgType)) {
-    OriginalArgType =
-        TypedPointerType::get(Type::getInt8Ty(F.getContext()),
-                              getPointerAddressSpace(OriginalArgType));
-  }
-  return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual);
+  return GR->getOrCreateSPIRVType(toTypedPointer(OriginalArgType), MIRBuilder,
+                                  ArgAccessQual);
 }
 
 static SPIRV::ExecutionModel::ExecutionModel
@@ -386,8 +379,8 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   Type *FRetTy = FTy->getReturnType();
   if (isUntypedPointerTy(FRetTy)) {
     if (Type *FRetElemTy = GR->findDeducedElementType(&F)) {
-      TypedPointerType *DerivedTy =
-          TypedPointerType::get(FRetElemTy, getPointerAddressSpace(FRetTy));
+      TypedPointerType *DerivedTy = TypedPointerType::get(
+          toTypedPointer(FRetElemTy), getPointerAddressSpace(FRetTy));
       GR->addReturnType(&F, DerivedTy);
       FRetTy = DerivedTy;
     }
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 5ef0be1cab722c16cb4a737f98a5b316861abe64..7b8e3230bf5534311a1d2eee50ba49367bf692a8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -50,6 +50,13 @@ void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
 } // namespace llvm
 
 namespace {
+
+inline MetadataAsValue *buildMD(Value *Arg) {
+  LLVMContext &Ctx = Arg->getContext();
+  return MetadataAsValue::get(
+      Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg)));
+}
+
 class SPIRVEmitIntrinsics
     : public ModulePass,
       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
@@ -61,9 +68,6 @@ class SPIRVEmitIntrinsics
   DenseMap<Instruction *, Type *> AggrConstTypes;
   DenseSet<Instruction *> AggrStores;
 
-  // a registry of created Intrinsic::spv_assign_ptr_type instructions
-  DenseMap<Value *, CallInst *> AssignPtrTypeInstr;
-
   // deduce element type of untyped pointers
   Type *deduceElementType(Value *I);
   Type *deduceElementTypeHelper(Value *I);
@@ -87,25 +91,24 @@ class SPIRVEmitIntrinsics
   CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
                             Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
                             IRBuilder<> &B) {
-    ConstantAsMetadata *CM = ValueAsMetadata::getConstant(Arg);
-    MDTuple *TyMD = MDNode::get(F->getContext(), CM);
-    MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
     SmallVector<Value *, 4> Args;
     Args.push_back(Arg2);
-    Args.push_back(VMD);
+    Args.push_back(buildMD(Arg));
     for (auto *Imm : Imms)
       Args.push_back(Imm);
     return B.CreateIntrinsic(IntrID, {Types}, Args);
   }
 
+  void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg);
   void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg);
+  void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType);
 
   void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
   void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
   void insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B);
   void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
-  void insertAssignTypeInstrForTargetExtTypes(TargetExtType *AssignedType,
-                                              Value *V, IRBuilder<> &B);
+  void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
+                                    IRBuilder<> &B);
   void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
                                         Type *ExpectedElementType,
                                         unsigned OperandToReplace,
@@ -218,15 +221,42 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
                        false);
 }
 
+void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
+                                          Value *Arg) {
+  Value *OfType = PoisonValue::get(Ty);
+  CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
+                                       {Arg->getType()}, OfType, Arg, {}, B);
+  GR->addAssignPtrTypeInstr(Arg, AssignCI);
+}
+
 void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,
                                          Value *Arg) {
-  CallInst *AssignPtrTyCI =
-      buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Arg->getType()},
-                      Constant::getNullValue(ElemTy), Arg,
-                      {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
-  GR->addDeducedElementType(AssignPtrTyCI, ElemTy);
+  Value *OfType = PoisonValue::get(ElemTy);
+  CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg);
+  if (AssignPtrTyCI == nullptr ||
+      AssignPtrTyCI->getParent()->getParent() != F) {
+    AssignPtrTyCI = buildIntrWithMD(
+        Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,
+        {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
+    GR->addDeducedElementType(AssignPtrTyCI, ElemTy);
+    GR->addDeducedElementType(Arg, ElemTy);
+    GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI);
+  } else {
+    updateAssignType(AssignPtrTyCI, Arg, OfType);
+  }
+}
+
+void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg,
+                                           Value *OfType) {
+  AssignCI->setArgOperand(1, buildMD(OfType));
+  if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() !=
+      Intrinsic::spv_assign_ptr_type)
+    return;
+
+  // update association with the pointee type
+  Type *ElemTy = OfType->getType();
+  GR->addDeducedElementType(AssignCI, ElemTy);
   GR->addDeducedElementType(Arg, ElemTy);
-  AssignPtrTypeInstr[Arg] = AssignPtrTyCI;
 }
 
 // Set element pointer type to the given value of ValueTy and tries to
@@ -513,19 +543,16 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) {
     if (!Ty) {
       GR->addDeducedElementType(Op, KnownElemTy);
       // check if there is existing Intrinsic::spv_assign_ptr_type instruction
-      auto It = AssignPtrTypeInstr.find(Op);
-      if (It == AssignPtrTypeInstr.end()) {
+      CallInst *AssignCI = GR->findAssignPtrTypeInstr(Op);
+      if (AssignCI == nullptr) {
         Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get());
         setInsertPointSkippingPhis(B, User ? User->getNextNode() : I);
         CallInst *CI =
             buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op,
                             {B.getInt32(getPointerAddressSpace(OpTy))}, B);
-        AssignPtrTypeInstr[Op] = CI;
+        GR->addAssignPtrTypeInstr(Op, CI);
       } else {
-        It->second->setArgOperand(
-            1,
-            MetadataAsValue::get(
-                Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OpTyVal))));
+        updateAssignType(AssignCI, Op, OpTyVal);
       }
     } else {
       if (auto *OpI = dyn_cast<Instruction>(Op)) {
@@ -540,9 +567,7 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) {
         B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
       }
       SmallVector<Type *, 2> Types = {OpTy, OpTy};
-      MetadataAsValue *VMD = MetadataAsValue::get(
-          Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OpTyVal)));
-      SmallVector<Value *, 2> Args = {Op, VMD,
+      SmallVector<Value *, 2> Args = {Op, buildMD(OpTyVal),
                                       B.getInt32(getPointerAddressSpace(OpTy))};
       CallInst *PtrCastI =
           B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
@@ -559,7 +584,9 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
     if (isAssignTypeInstr(U)) {
       B.SetInsertPoint(U);
       SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
-      B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
+      CallInst *AssignCI =
+          B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
+      GR->addAssignPtrTypeInstr(New, AssignCI);
       U->eraseFromParent();
     } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
                isa<CallInst>(U)) {
@@ -667,8 +694,7 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
   Constant *TyC = UndefValue::get(IA->getFunctionType());
   MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
   SmallVector<Value *> Args = {
-      MetadataAsValue::get(Ctx,
-                           MDNode::get(Ctx, ValueAsMetadata::getConstant(TyC))),
+      buildMD(TyC),
       MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))};
   for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
     Args.push_back(Call.getArgOperand(OpIdx));
@@ -751,33 +777,39 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
   return NewI;
 }
 
-void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes(
+void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
     TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
-  // Do not emit spv_assign_type if the V is of the AssignedType already.
-  if (V->getType() == AssignedType)
-    return;
+  Type *VTy = V->getType();
 
-  // Do not emit spv_assign_type if there is one already targetting V. If the
-  // found spv_assign_type assigns a type different than AssignedType, report an
-  // error. Builtin types cannot be redeclared or casted.
-  for (auto User : V->users()) {
-    auto *II = dyn_cast<IntrinsicInst>(User);
-    if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_type)
-      continue;
+  // A couple of sanity checks.
+  assert(isPointerTy(VTy) && "Expect a pointer type!");
+  if (auto PType = dyn_cast<TypedPointerType>(VTy))
+    if (PType->getElementType() != AssignedType)
+      report_fatal_error("Unexpected pointer element type!");
 
-    MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1));
-    Type *BuiltinType =
-        dyn_cast<ConstantAsMetadata>(VMD->getMetadata())->getType();
-    if (BuiltinType != AssignedType)
-      report_fatal_error("Type mismatch " + BuiltinType->getTargetExtName() +
-                             "/" + AssignedType->getTargetExtName() +
-                             " for value " + V->getName(),
-                         false);
+  CallInst *AssignCI = GR->findAssignPtrTypeInstr(V);
+  if (!AssignCI) {
+    buildAssignType(B, AssignedType, V);
     return;
   }
 
-  Constant *Const = UndefValue::get(AssignedType);
-  buildIntrWithMD(Intrinsic::spv_assign_type, {V->getType()}, Const, V, {}, B);
+  Type *CurrentType =
+      dyn_cast<ConstantAsMetadata>(
+          cast<MetadataAsValue>(AssignCI->getOperand(1))->getMetadata())
+          ->getType();
+  if (CurrentType == AssignedType)
+    return;
+
+  // Builtin types cannot be redeclared or casted.
+  if (CurrentType->isTargetExtTy())
+    report_fatal_error("Type mismatch " + CurrentType->getTargetExtName() +
+                           "/" + AssignedType->getTargetExtName() +
+                           " for value " + V->getName(),
+                       false);
+
+  // Our previous guess about the type seems to be wrong, let's update
+  // inferred type according to a new, more precise type information.
+  updateAssignType(AssignCI, V, PoisonValue::get(AssignedType));
 }
 
 void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
@@ -794,12 +826,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
     return;
 
   setInsertPointSkippingPhis(B, I);
-  Constant *ExpectedElementTypeConst =
-      Constant::getNullValue(ExpectedElementType);
-  ConstantAsMetadata *CM =
-      ValueAsMetadata::getConstant(ExpectedElementTypeConst);
-  MDTuple *TyMD = MDNode::get(F->getContext(), CM);
-  MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
+  MetadataAsValue *VMD = buildMD(PoisonValue::get(ExpectedElementType));
   unsigned AddressSpace = getPointerAddressSpace(Pointer->getType());
   bool FirstPtrCastOrAssignPtrType = true;
 
@@ -845,12 +872,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
   // spv_assign_ptr_type instead.
   if (FirstPtrCastOrAssignPtrType &&
       (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
-    CallInst *CI = buildIntrWithMD(
-        Intrinsic::spv_assign_ptr_type, {Pointer->getType()},
-        ExpectedElementTypeConst, Pointer, {B.getInt32(AddressSpace)}, B);
-    GR->addDeducedElementType(CI, ExpectedElementType);
-    GR->addDeducedElementType(Pointer, ExpectedElementType);
-    AssignPtrTypeInstr[Pointer] = CI;
+    buildAssignPtr(B, ExpectedElementType, Pointer);
     return;
   }
 
@@ -929,8 +951,7 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
 
   for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
     Value *ArgOperand = CI->getArgOperand(OpIdx);
-    if (!isa<PointerType>(ArgOperand->getType()) &&
-        !isa<TypedPointerType>(ArgOperand->getType()))
+    if (!isPointerTy(ArgOperand->getType()))
       continue;
 
     // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
@@ -952,8 +973,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
       continue;
 
     if (ExpectedType->isTargetExtTy())
-      insertAssignTypeInstrForTargetExtTypes(cast<TargetExtType>(ExpectedType),
-                                             ArgOperand, B);
+      insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType),
+                                   ArgOperand, B);
     else
       replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B);
   }
@@ -1140,12 +1161,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
 
   setInsertPointAfterDef(B, I);
   Type *ElemTy = deduceElementType(I);
-  Constant *EltTyConst = UndefValue::get(ElemTy);
-  unsigned AddressSpace = getPointerAddressSpace(I->getType());
-  CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()},
-                                 EltTyConst, I, {B.getInt32(AddressSpace)}, B);
-  GR->addDeducedElementType(CI, ElemTy);
-  AssignPtrTypeInstr[I] = CI;
+  buildAssignPtr(B, ElemTy, I);
 }
 
 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
@@ -1164,20 +1180,32 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
         TypeToAssign = It->second;
       }
     }
-    Constant *Const = UndefValue::get(TypeToAssign);
-    buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I, {}, B);
+    buildAssignType(B, TypeToAssign, I);
   }
   for (const auto &Op : I->operands()) {
     if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
         // Check GetElementPtrConstantExpr case.
         (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
       setInsertPointSkippingPhis(B, I);
-      if (isa<UndefValue>(Op) && Op->getType()->isAggregateType())
-        buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op,
-                        UndefValue::get(B.getInt32Ty()), {}, B);
-      else if (!isa<Instruction>(Op))
-        buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op, {},
-                        B);
+      Type *OpTy = Op->getType();
+      if (isa<UndefValue>(Op) && OpTy->isAggregateType()) {
+        CallInst *AssignCI =
+            buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op,
+                            UndefValue::get(B.getInt32Ty()), {}, B);
+        GR->addAssignPtrTypeInstr(Op, AssignCI);
+      } else if (!isa<Instruction>(Op)) {
+        Type *OpTy = Op->getType();
+        if (auto PType = dyn_cast<TypedPointerType>(OpTy)) {
+          buildAssignPtr(B, PType->getElementType(), Op);
+        } else if (isPointerTy(OpTy)) {
+          Type *ElemTy = GR->findDeducedElementType(Op);
+          buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op), Op);
+        } else {
+          CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
+                                               {OpTy}, Op, Op, {}, B);
+          GR->addAssignPtrTypeInstr(Op, AssignCI);
+        }
+      }
     }
   }
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index f96c3a2b0a77060c0ffa39290611fadd227d2a70..d434e0b5efbccba16949ebce8a314e93d6dff551 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -742,8 +742,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeStruct(const StructType *Ty,
                                                 bool EmitIR) {
   SmallVector<Register, 4> FieldTypes;
   for (const auto &Elem : Ty->elements()) {
-    SPIRVType *ElemTy =
-        findSPIRVType(toTypedPointer(Elem, Ty->getContext()), MIRBuilder);
+    SPIRVType *ElemTy = findSPIRVType(toTypedPointer(Elem), MIRBuilder);
     assert(ElemTy && ElemTy->getOpcode() != SPIRV::OpTypeVoid &&
            "Invalid struct element type");
     FieldTypes.push_back(getSPIRVTypeID(ElemTy));
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index ef0973d03d155b578c7092b748231e5681cdb1b5..db01f68f48de9744d07be3b350c02905e77e072a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -73,8 +73,11 @@ class SPIRVGlobalRegistry {
   // untyped pointers.
   DenseMap<Value *, Type *> DeducedElTys;
   // Maps composite values to deduced types where untyped pointers are replaced
-  // with typed ones
+  // with typed ones.
   DenseMap<Value *, Type *> DeducedNestedTys;
+  // Maps values to "assign type" calls, thus being a registry of created
+  // Intrinsic::spv_assign_ptr_type instructions.
+  DenseMap<Value *, CallInst *> AssignPtrTypeInstr;
 
   // Add a new OpTypeXXX instruction without checking for duplicates.
   SPIRVType *createSPIRVType(const Type *Type, MachineIRBuilder &MIRBuilder,
@@ -149,6 +152,17 @@ public:
     return It == FunResPointerTypes.end() ? nullptr : It->second;
   }
 
+  // A registry of "assign type" records:
+  // - Add a record.
+  void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI) {
+    AssignPtrTypeInstr[Val] = AssignPtrTyCI;
+  }
+  // - Find a record.
+  CallInst *findAssignPtrTypeInstr(const Value *Val) {
+    auto It = AssignPtrTypeInstr.find(Val);
+    return It == AssignPtrTypeInstr.end() ? nullptr : It->second;
+  }
+
   // Deduced element types of untyped pointers and composites:
   // - Add a record to the map of deduced element types.
   void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 517a9b490ebada1e719c52adee33c0b8082fe708..db83172f7fa9c4a5eabae13e588b9ec9137c775d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1086,7 +1086,11 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
 
   // don't generate a cast between identical storage classes
   if (SrcSC == DstSC)
-    return true;
+    return BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                   TII.get(TargetOpcode::COPY))
+        .addDef(ResVReg)
+        .addUse(SrcPtr)
+        .constrainAllUses(TII, TRI, RBI);
 
   // Casting from an eligible pointer to Generic.
   if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
@@ -1121,10 +1125,13 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
   if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
     return selectUnOp(ResVReg, ResType, I,
                       SPIRV::OpCrossWorkgroupCastToPtrINTEL);
+  if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic)
+    return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
+  if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC))
+    return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
 
-  // TODO Should this case just be disallowed completely?
-  // We're casting 2 other arbitrary address spaces, so have to bitcast.
-  return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
+  // Bitcast for pointers requires that the address spaces must match
+  return false;
 }
 
 static unsigned getFCmpOpcode(unsigned PredNum) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
index 4b0e7c421df3e57583aa3579ca2ba5248d63c422..2744c25d1bc754a7af31e42eaccf2c9361e4c5ab 100644
--- a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp
@@ -106,7 +106,7 @@ public:
                             const std::unordered_set<BasicBlock *> ToReplace,
                             BasicBlock *NewTarget) {
     auto *T = BB->getTerminator();
-    if (auto *RI = dyn_cast<ReturnInst>(T))
+    if (isa<ReturnInst>(T))
       return;
 
     if (auto *BI = dyn_cast<BranchInst>(T)) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 61f99f8d85269561db3c7816d3e0cf6598c601d2..30a6c474f467a0bdceb5f27f0519f5d205844b6b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -95,7 +95,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
     if (ReqCaps.size() == 1) {
       auto Cap = ReqCaps[0];
       if (Reqs.isCapabilityAvailable(Cap))
-        return {true, {Cap}, {}, ReqMinVer, ReqMaxVer};
+        return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
     } else {
       // By SPIR-V specification: "If an instruction, enumerant, or other
       // feature specifies multiple enabling capabilities, only one such
@@ -110,7 +110,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
       for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
         auto Cap = UseCaps[i];
         if (i == Sz - 1 || !AvoidCaps.S.contains(Cap))
-          return {true, {Cap}, {}, ReqMinVer, ReqMaxVer};
+          return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
       }
     }
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 3d536085b78aa61fcf75d4b433a0973e09b8e27b..adc5b36af6f182e204e1a0d06e8775ec30589935 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -166,8 +166,8 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
       }
       Register Def = MI.getOperand(0).getReg();
       Register Source = MI.getOperand(2).getReg();
-      SPIRVType *BaseTy = GR->getOrCreateSPIRVType(
-          getMDOperandAsType(MI.getOperand(3).getMetadata(), 0), MIB);
+      Type *ElemTy = getMDOperandAsType(MI.getOperand(3).getMetadata(), 0);
+      SPIRVType *BaseTy = GR->getOrCreateSPIRVType(ElemTy, MIB);
       SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(
           BaseTy, MI, *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo(),
           addressSpaceToStorageClass(MI.getOperand(4).getImm(), *ST));
@@ -220,7 +220,7 @@ static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR,
         MIB.setInsertPt(*MI->getParent(), MI);
         const GlobalValue *Global = MI->getOperand(1).getGlobal();
         Type *ElementTy = GR->getDeducedGlobalValueType(Global);
-        auto *Ty = TypedPointerType::get(ElementTy,
+        auto *Ty = TypedPointerType::get(toTypedPointer(ElementTy),
                                          Global->getType()->getAddressSpace());
         SpirvTy = GR->getOrCreateSPIRVType(Ty, MIB);
         break;
@@ -409,15 +409,16 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
       if (isSpvIntrinsic(MI, Intrinsic::spv_assign_ptr_type)) {
         Register Reg = MI.getOperand(1).getReg();
         MIB.setInsertPt(*MI.getParent(), MI.getIterator());
-        SPIRVType *BaseTy = GR->getOrCreateSPIRVType(
-            getMDOperandAsType(MI.getOperand(2).getMetadata(), 0), MIB);
+        Type *ElementTy = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0);
+        SPIRVType *BaseTy = GR->getOrCreateSPIRVType(ElementTy, MIB);
         SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(
             BaseTy, MI, *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo(),
             addressSpaceToStorageClass(MI.getOperand(3).getImm(), *ST));
         MachineInstr *Def = MRI.getVRegDef(Reg);
         assert(Def && "Expecting an instruction that defines the register");
         // G_GLOBAL_VALUE already has type info.
-        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
+        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
+            Def->getOpcode() != SPIRV::ASSIGN_TYPE)
           insertAssignInstr(Reg, nullptr, AssignedPtrType, GR, MIB,
                             MF.getRegInfo());
         ToErase.push_back(&MI);
@@ -427,7 +428,8 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
         MachineInstr *Def = MRI.getVRegDef(Reg);
         assert(Def && "Expecting an instruction that defines the register");
         // G_GLOBAL_VALUE already has type info.
-        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
+        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
+            Def->getOpcode() != SPIRV::ASSIGN_TYPE)
           insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
         ToErase.push_back(&MI);
       } else if (MIOp == TargetOpcode::G_CONSTANT ||
@@ -451,6 +453,7 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
           Ty = TargetExtIt == TargetExtConstTypes.end()
                    ? MI.getOperand(1).getCImm()->getType()
                    : TargetExtIt->second;
+          GR->add(MI.getOperand(1).getCImm(), &MF, Reg);
         } else if (MIOp == TargetOpcode::G_FCONSTANT) {
           Ty = MI.getOperand(1).getFPImm()->getType();
         } else {
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 65b48c8acf6ab735f80f9acd428896a860b92bc8..318c5cebb7a43a1c0bf3f305f30b8e3ffe0488be 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -689,34 +689,34 @@ class StorageClass<string name, bits<32> value> {
   bits<32> Value = value;
 }
 
-multiclass StorageClassOperand<bits<32> value, list<Capability> reqCapabilities> {
+multiclass StorageClassOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : StorageClass<NAME, value>;
-  defm : SymbolicOperandWithRequirements<StorageClassOperand, value, NAME, 0, 0, [], reqCapabilities>;
-}
-
-defm UniformConstant : StorageClassOperand<0, []>;
-defm Input : StorageClassOperand<1, []>;
-defm Uniform : StorageClassOperand<2, [Shader]>;
-defm Output : StorageClassOperand<3, [Shader]>;
-defm Workgroup : StorageClassOperand<4, []>;
-defm CrossWorkgroup : StorageClassOperand<5, []>;
-defm Private : StorageClassOperand<6, [Shader]>;
-defm Function : StorageClassOperand<7, []>;
-defm Generic : StorageClassOperand<8, [GenericPointer]>;
-defm PushConstant : StorageClassOperand<9, [Shader]>;
-defm AtomicCounter : StorageClassOperand<10, [AtomicStorage]>;
-defm Image : StorageClassOperand<11, []>;
-defm StorageBuffer : StorageClassOperand<12, [Shader]>;
-defm CallableDataNV : StorageClassOperand<5328, [RayTracingNV]>;
-defm IncomingCallableDataNV : StorageClassOperand<5329, [RayTracingNV]>;
-defm RayPayloadNV : StorageClassOperand<5338, [RayTracingNV]>;
-defm HitAttributeNV : StorageClassOperand<5339, [RayTracingNV]>;
-defm IncomingRayPayloadNV : StorageClassOperand<5342, [RayTracingNV]>;
-defm ShaderRecordBufferNV : StorageClassOperand<5343, [RayTracingNV]>;
-defm PhysicalStorageBufferEXT : StorageClassOperand<5349, [PhysicalStorageBufferAddressesEXT]>;
-defm CodeSectionINTEL : StorageClassOperand<5605, [FunctionPointersINTEL]>;
-defm DeviceOnlyINTEL : StorageClassOperand<5936, [USMStorageClassesINTEL]>;
-defm HostOnlyINTEL : StorageClassOperand<5937, [USMStorageClassesINTEL]>;
+  defm : SymbolicOperandWithRequirements<StorageClassOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm UniformConstant : StorageClassOperand<0, [], []>;
+defm Input : StorageClassOperand<1, [], []>;
+defm Uniform : StorageClassOperand<2, [], [Shader]>;
+defm Output : StorageClassOperand<3, [], [Shader]>;
+defm Workgroup : StorageClassOperand<4, [], []>;
+defm CrossWorkgroup : StorageClassOperand<5, [], []>;
+defm Private : StorageClassOperand<6, [], [Shader]>;
+defm Function : StorageClassOperand<7, [], []>;
+defm Generic : StorageClassOperand<8, [], [GenericPointer]>;
+defm PushConstant : StorageClassOperand<9, [], [Shader]>;
+defm AtomicCounter : StorageClassOperand<10, [], [AtomicStorage]>;
+defm Image : StorageClassOperand<11, [], []>;
+defm StorageBuffer : StorageClassOperand<12, [], [Shader]>;
+defm CallableDataNV : StorageClassOperand<5328, [], [RayTracingNV]>;
+defm IncomingCallableDataNV : StorageClassOperand<5329, [], [RayTracingNV]>;
+defm RayPayloadNV : StorageClassOperand<5338, [], [RayTracingNV]>;
+defm HitAttributeNV : StorageClassOperand<5339, [], [RayTracingNV]>;
+defm IncomingRayPayloadNV : StorageClassOperand<5342, [], [RayTracingNV]>;
+defm ShaderRecordBufferNV : StorageClassOperand<5343, [], [RayTracingNV]>;
+defm PhysicalStorageBufferEXT : StorageClassOperand<5349, [], [PhysicalStorageBufferAddressesEXT]>;
+defm CodeSectionINTEL : StorageClassOperand<5605, [SPV_INTEL_function_pointers], [FunctionPointersINTEL]>;
+defm DeviceOnlyINTEL : StorageClassOperand<5936, [SPV_INTEL_usm_storage_classes], [USMStorageClassesINTEL]>;
+defm HostOnlyINTEL : StorageClassOperand<5937, [SPV_INTEL_usm_storage_classes], [USMStorageClassesINTEL]>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Dim enum values and at the same time
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index a6823a8ba3230e90844b3c9b1915d8fb43b5a804..52fc6f33b4ef14b338ff79ccb2c1e3996bbe15ff 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -56,6 +56,10 @@ static std::string computeDataLayout(const Triple &TT) {
   if (Arch == Triple::spirv32)
     return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-"
            "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1";
+  if (TT.getVendor() == Triple::VendorType::AMD &&
+      TT.getOS() == Triple::OSType::AMDHSA)
+    return "e-i64:64-v16:16-v24:32-v32:32-v48:64-"
+           "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0";
   return "e-i64:64-v16:16-v24:32-v32:32-v48:64-"
          "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1";
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index c20f3546a3e5566a9e3f873908c051e96b683b09..c1b90b0e9d884099ee03e3f57989077e28a901bf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -280,7 +280,7 @@ bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
 
 Type *getMDOperandAsType(const MDNode *N, unsigned I) {
   Type *ElementTy = cast<ValueAsMetadata>(N->getOperand(I))->getType();
-  return toTypedPointer(ElementTy, N->getContext());
+  return toTypedPointer(ElementTy);
 }
 
 // The set of names is borrowed from the SPIR-V translator.
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 33cb509dc4a597fc98c1dae311b33658124d78e2..c131eecb1c13754df3d2152791afaa929377ff9a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -113,7 +113,7 @@ inline bool isTypedPointerTy(const Type *T) {
 
 // True if this is an instance of PointerType.
 inline bool isUntypedPointerTy(const Type *T) {
-  return T->getTypeID() == Type::PointerTyID;
+  return T && T->getTypeID() == Type::PointerTyID;
 }
 
 // True if this is an instance of PointerType or TypedPointerType.
@@ -153,9 +153,9 @@ inline Type *reconstructFunctionType(Function *F) {
   return FunctionType::get(F->getReturnType(), ArgTys, F->isVarArg());
 }
 
-inline Type *toTypedPointer(Type *Ty, LLVMContext &Ctx) {
+inline Type *toTypedPointer(Type *Ty) {
   return isUntypedPointerTy(Ty)
-             ? TypedPointerType::get(IntegerType::getInt8Ty(Ctx),
+             ? TypedPointerType::get(IntegerType::getInt8Ty(Ty->getContext()),
                                      getPointerAddressSpace(Ty))
              : Ty;
 }
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index 185b2fe90c6cb643eefdd048f0929aa4e2d8b0a1..f0a3a4e88b30ced51f0d47d30a73fb3c89a928f5 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -91,6 +91,8 @@ class SparcAsmParser : public MCTargetAsmParser {
 
   ParseStatus parseASITag(OperandVector &Operands);
 
+  ParseStatus parsePrefetchTag(OperandVector &Operands);
+
   template <TailRelocKind Kind>
   ParseStatus parseTailRelocSym(OperandVector &Operands);
 
@@ -209,7 +211,8 @@ private:
     k_Immediate,
     k_MemoryReg,
     k_MemoryImm,
-    k_ASITag
+    k_ASITag,
+    k_PrefetchTag,
   } Kind;
 
   SMLoc StartLoc, EndLoc;
@@ -240,6 +243,7 @@ private:
     struct ImmOp Imm;
     struct MemOp Mem;
     unsigned ASI;
+    unsigned Prefetch;
   };
 
 public:
@@ -253,6 +257,7 @@ public:
   bool isMEMri() const { return Kind == k_MemoryImm; }
   bool isMembarTag() const { return Kind == k_Immediate; }
   bool isASITag() const { return Kind == k_ASITag; }
+  bool isPrefetchTag() const { return Kind == k_PrefetchTag; }
   bool isTailRelocSym() const { return Kind == k_Immediate; }
 
   bool isCallTarget() const {
@@ -337,6 +342,11 @@ public:
     return ASI;
   }
 
+  unsigned getPrefetchTag() const {
+    assert((Kind == k_PrefetchTag) && "Invalid access!");
+    return Prefetch;
+  }
+
   /// getStartLoc - Get the location of the first token of this operand.
   SMLoc getStartLoc() const override {
     return StartLoc;
@@ -360,6 +370,9 @@ public:
     case k_ASITag:
       OS << "ASI tag: " << getASITag() << "\n";
       break;
+    case k_PrefetchTag:
+      OS << "Prefetch tag: " << getPrefetchTag() << "\n";
+      break;
     }
   }
 
@@ -416,6 +429,11 @@ public:
     Inst.addOperand(MCOperand::createImm(getASITag()));
   }
 
+  void addPrefetchTagOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createImm(getPrefetchTag()));
+  }
+
   void addMembarTagOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCExpr *Expr = getImm();
@@ -469,6 +487,15 @@ public:
     return Op;
   }
 
+  static std::unique_ptr<SparcOperand> CreatePrefetchTag(unsigned Val, SMLoc S,
+                                                         SMLoc E) {
+    auto Op = std::make_unique<SparcOperand>(k_PrefetchTag);
+    Op->Prefetch = Val;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
   static bool MorphToIntPairReg(SparcOperand &Op) {
     unsigned Reg = Op.getReg();
     assert(Op.Reg.Kind == rk_IntReg);
@@ -1088,6 +1115,44 @@ ParseStatus SparcAsmParser::parseASITag(OperandVector &Operands) {
   return ParseStatus::Success;
 }
 
+ParseStatus SparcAsmParser::parsePrefetchTag(OperandVector &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+  SMLoc E = Parser.getTok().getEndLoc();
+  int64_t PrefetchVal = 0;
+
+  switch (getLexer().getKind()) {
+  case AsmToken::LParen:
+  case AsmToken::Integer:
+  case AsmToken::Identifier:
+  case AsmToken::Plus:
+  case AsmToken::Minus:
+  case AsmToken::Tilde:
+    if (getParser().parseAbsoluteExpression(PrefetchVal) ||
+        !isUInt<5>(PrefetchVal))
+      return Error(S, "invalid prefetch number, must be between 0 and 31");
+    break;
+  case AsmToken::Hash: {
+    SMLoc TagStart = getLexer().peekTok(false).getLoc();
+    Parser.Lex(); // Eat the '#'.
+    const StringRef PrefetchName = Parser.getTok().getString();
+    const SparcPrefetchTag::PrefetchTag *PrefetchTag =
+        SparcPrefetchTag::lookupPrefetchTagByName(PrefetchName);
+    Parser.Lex(); // Eat the identifier token.
+
+    if (!PrefetchTag)
+      return Error(TagStart, "unknown prefetch tag");
+
+    PrefetchVal = PrefetchTag->Encoding;
+    break;
+  }
+  default:
+    return ParseStatus::NoMatch;
+  }
+
+  Operands.push_back(SparcOperand::CreatePrefetchTag(PrefetchVal, S, E));
+  return ParseStatus::Success;
+}
+
 ParseStatus SparcAsmParser::parseCallTarget(OperandVector &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
@@ -1384,12 +1449,11 @@ MCRegister SparcAsmParser::matchRegisterName(const AsmToken &Tok,
   }
 
   // JPS1 extension - aliases for ASRs
-  // Section A.51 - Read State Register
+  // Section 5.2.11 - Ancillary State Registers (ASRs)
   if (Name == "pcr") {
     RegKind = SparcOperand::rk_Special;
     return SP::ASR16;
   }
-
   if (Name == "pic") {
     RegKind = SparcOperand::rk_Special;
     return SP::ASR17;
@@ -1402,6 +1466,14 @@ MCRegister SparcAsmParser::matchRegisterName(const AsmToken &Tok,
     RegKind = SparcOperand::rk_Special;
     return SP::ASR19;
   }
+  if (Name == "set_softint") {
+    RegKind = SparcOperand::rk_Special;
+    return SP::ASR20;
+  }
+  if (Name == "clear_softint") {
+    RegKind = SparcOperand::rk_Special;
+    return SP::ASR21;
+  }
   if (Name == "softint") {
     RegKind = SparcOperand::rk_Special;
     return SP::ASR22;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index 240f5396855c832438c7b92e0421286eb324bc41..cb7414fddd29f0efe4025abe1ef18cbdac4df7db 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -323,9 +323,11 @@ namespace {
 
     bool writeNopData(raw_ostream &OS, uint64_t Count,
                       const MCSubtargetInfo *STI) const override {
-      // Cannot emit NOP with size not multiple of 32 bits.
-      if (Count % 4 != 0)
-        return false;
+
+      // If the count is not 4-byte aligned, we must be writing data into the
+      // text section (otherwise we have unaligned instructions, and thus have
+      // far bigger problems), so just write zeros instead.
+      OS.write_zeros(Count % 4);
 
       uint64_t NumNops = Count / 4;
       for (uint64_t i = 0; i != NumNops; ++i)
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
index ef776485047161fd77fcc27b8a1dffb0397b7e17..5b407a8b6f54a0fc7cba0a3a9515953ca184096a 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
@@ -253,3 +253,14 @@ void SparcInstPrinter::printASITag(const MCInst *MI, int opNum,
   else
     O << Imm;
 }
+
+void SparcInstPrinter::printPrefetchTag(const MCInst *MI, int opNum,
+                                        const MCSubtargetInfo &STI,
+                                        raw_ostream &O) {
+  unsigned Imm = MI->getOperand(opNum).getImm();
+  auto PrefetchTag = SparcPrefetchTag::lookupPrefetchTagByEncoding(Imm);
+  if (PrefetchTag)
+    O << '#' << PrefetchTag->Name;
+  else
+    O << Imm;
+}
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
index cb691a3420da7f8c023e32cf8c740921ac3d4fc6..207a970228058daeca30725cd1f508bd13fde5e9 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
@@ -56,6 +56,8 @@ public:
                       raw_ostream &O);
   void printASITag(const MCInst *MI, int opNum, const MCSubtargetInfo &STI,
                    raw_ostream &O);
+  void printPrefetchTag(const MCInst *MI, int opNum, const MCSubtargetInfo &STI,
+                        raw_ostream &O);
 };
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
index 522a8877a1c60a8139109fdf4982802fd8bce07d..4688837e435e5cd18bddcafc90aa99ff98108765 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
@@ -93,44 +93,46 @@ bool SparcMCExpr::printVariantKind(raw_ostream &OS, VariantKind Kind)
 SparcMCExpr::VariantKind SparcMCExpr::parseVariantKind(StringRef name)
 {
   return StringSwitch<SparcMCExpr::VariantKind>(name)
-    .Case("lo",  VK_Sparc_LO)
-    .Case("hi",  VK_Sparc_HI)
-    .Case("h44", VK_Sparc_H44)
-    .Case("m44", VK_Sparc_M44)
-    .Case("l44", VK_Sparc_L44)
-    .Case("hh",  VK_Sparc_HH)
-    .Case("hm",  VK_Sparc_HM)
-    .Case("lm",  VK_Sparc_LM)
-    .Case("pc22",  VK_Sparc_PC22)
-    .Case("pc10",  VK_Sparc_PC10)
-    .Case("got22", VK_Sparc_GOT22)
-    .Case("got10", VK_Sparc_GOT10)
-    .Case("got13", VK_Sparc_GOT13)
-    .Case("r_disp32",   VK_Sparc_R_DISP32)
-    .Case("tgd_hi22",   VK_Sparc_TLS_GD_HI22)
-    .Case("tgd_lo10",   VK_Sparc_TLS_GD_LO10)
-    .Case("tgd_add",    VK_Sparc_TLS_GD_ADD)
-    .Case("tgd_call",   VK_Sparc_TLS_GD_CALL)
-    .Case("tldm_hi22",  VK_Sparc_TLS_LDM_HI22)
-    .Case("tldm_lo10",  VK_Sparc_TLS_LDM_LO10)
-    .Case("tldm_add",   VK_Sparc_TLS_LDM_ADD)
-    .Case("tldm_call",  VK_Sparc_TLS_LDM_CALL)
-    .Case("tldo_hix22", VK_Sparc_TLS_LDO_HIX22)
-    .Case("tldo_lox10", VK_Sparc_TLS_LDO_LOX10)
-    .Case("tldo_add",   VK_Sparc_TLS_LDO_ADD)
-    .Case("tie_hi22",   VK_Sparc_TLS_IE_HI22)
-    .Case("tie_lo10",   VK_Sparc_TLS_IE_LO10)
-    .Case("tie_ld",     VK_Sparc_TLS_IE_LD)
-    .Case("tie_ldx",    VK_Sparc_TLS_IE_LDX)
-    .Case("tie_add",    VK_Sparc_TLS_IE_ADD)
-    .Case("tle_hix22",  VK_Sparc_TLS_LE_HIX22)
-    .Case("tle_lox10",  VK_Sparc_TLS_LE_LOX10)
-    .Case("hix",        VK_Sparc_HIX22)
-    .Case("lox",        VK_Sparc_LOX10)
-    .Case("gdop_hix22", VK_Sparc_GOTDATA_HIX22)
-    .Case("gdop_lox10", VK_Sparc_GOTDATA_LOX10)
-    .Case("gdop",       VK_Sparc_GOTDATA_OP)
-    .Default(VK_Sparc_None);
+      .Case("lo", VK_Sparc_LO)
+      .Case("hi", VK_Sparc_HI)
+      .Case("h44", VK_Sparc_H44)
+      .Case("m44", VK_Sparc_M44)
+      .Case("l44", VK_Sparc_L44)
+      .Case("hh", VK_Sparc_HH)
+      .Case("uhi", VK_Sparc_HH) // Nonstandard GNU extension
+      .Case("hm", VK_Sparc_HM)
+      .Case("ulo", VK_Sparc_HM) // Nonstandard GNU extension
+      .Case("lm", VK_Sparc_LM)
+      .Case("pc22", VK_Sparc_PC22)
+      .Case("pc10", VK_Sparc_PC10)
+      .Case("got22", VK_Sparc_GOT22)
+      .Case("got10", VK_Sparc_GOT10)
+      .Case("got13", VK_Sparc_GOT13)
+      .Case("r_disp32", VK_Sparc_R_DISP32)
+      .Case("tgd_hi22", VK_Sparc_TLS_GD_HI22)
+      .Case("tgd_lo10", VK_Sparc_TLS_GD_LO10)
+      .Case("tgd_add", VK_Sparc_TLS_GD_ADD)
+      .Case("tgd_call", VK_Sparc_TLS_GD_CALL)
+      .Case("tldm_hi22", VK_Sparc_TLS_LDM_HI22)
+      .Case("tldm_lo10", VK_Sparc_TLS_LDM_LO10)
+      .Case("tldm_add", VK_Sparc_TLS_LDM_ADD)
+      .Case("tldm_call", VK_Sparc_TLS_LDM_CALL)
+      .Case("tldo_hix22", VK_Sparc_TLS_LDO_HIX22)
+      .Case("tldo_lox10", VK_Sparc_TLS_LDO_LOX10)
+      .Case("tldo_add", VK_Sparc_TLS_LDO_ADD)
+      .Case("tie_hi22", VK_Sparc_TLS_IE_HI22)
+      .Case("tie_lo10", VK_Sparc_TLS_IE_LO10)
+      .Case("tie_ld", VK_Sparc_TLS_IE_LD)
+      .Case("tie_ldx", VK_Sparc_TLS_IE_LDX)
+      .Case("tie_add", VK_Sparc_TLS_IE_ADD)
+      .Case("tle_hix22", VK_Sparc_TLS_LE_HIX22)
+      .Case("tle_lox10", VK_Sparc_TLS_LE_LOX10)
+      .Case("hix", VK_Sparc_HIX22)
+      .Case("lox", VK_Sparc_LOX10)
+      .Case("gdop_hix22", VK_Sparc_GOTDATA_HIX22)
+      .Case("gdop_lox10", VK_Sparc_GOTDATA_LOX10)
+      .Case("gdop", VK_Sparc_GOTDATA_OP)
+      .Default(VK_Sparc_None);
 }
 
 Sparc::Fixups SparcMCExpr::getFixupKind(SparcMCExpr::VariantKind Kind) {
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
index fb634ccb280dfe09cc7c91f3e8c60c24560f5d44..ad6ca0911adb98edf67824051cc4ec0d9fdfe0ed 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
@@ -26,6 +26,11 @@ namespace SparcASITag {
 #define GET_ASITagsList_IMPL
 #include "SparcGenSearchableTables.inc"
 } // end namespace SparcASITag
+
+namespace SparcPrefetchTag {
+#define GET_PrefetchTagsList_IMPL
+#include "SparcGenSearchableTables.inc"
+} // end namespace SparcPrefetchTag
 } // end namespace llvm
 
 using namespace llvm;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
index fd76627aa06756dbb58338732ed7949f6819e767..a2a9f7474c3f914139ca55df0d1cb30218c1dd71 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
@@ -48,6 +48,17 @@ struct ASITag {
 #define GET_ASITagsList_DECL
 #include "SparcGenSearchableTables.inc"
 } // end namespace SparcASITag
+
+// Defines symbolic names for Sparc v9 prefetch tag names.
+namespace SparcPrefetchTag {
+struct PrefetchTag {
+  const char *Name;
+  unsigned Encoding;
+};
+
+#define GET_PrefetchTagsList_DECL
+#include "SparcGenSearchableTables.inc"
+} // end namespace SparcPrefetchTag
 } // End llvm namespace
 
 // Defines symbolic names for Sparc registers.  This defines a mapping from
diff --git a/llvm/lib/Target/Sparc/Sparc.td b/llvm/lib/Target/Sparc/Sparc.td
index 45cf985cfa0627051b5b6174d79e0a22d5d30689..65f372f4376b13e0c1add18e576df323fac146a4 100644
--- a/llvm/lib/Target/Sparc/Sparc.td
+++ b/llvm/lib/Target/Sparc/Sparc.td
@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 include "llvm/Target/Target.td"
+include "llvm/TableGen/SearchableTable.td"
 
 //===----------------------------------------------------------------------===//
 // SPARC Subtarget features.
@@ -91,6 +92,7 @@ foreach i = 0 ... 5 in
 //===----------------------------------------------------------------------===//
 
 include "SparcASITags.td"
+include "SparcPrefetchTags.td"
 include "SparcRegisterInfo.td"
 include "SparcCallingConv.td"
 include "SparcSchedule.td"
diff --git a/llvm/lib/Target/Sparc/SparcASITags.td b/llvm/lib/Target/Sparc/SparcASITags.td
index 115e41bfe0333890910734fcf31a76b13908be0b..4b2d17b77ee59845524eb2bf30da5c39717323b8 100644
--- a/llvm/lib/Target/Sparc/SparcASITags.td
+++ b/llvm/lib/Target/Sparc/SparcASITags.td
@@ -11,8 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-include "llvm/TableGen/SearchableTable.td"
-
 class ASITag<string name, string alt_name, bits<8> op> {
   string Name = name;
   // A maximum of one alias is supported right now.
diff --git a/llvm/lib/Target/Sparc/SparcInstrAliases.td b/llvm/lib/Target/Sparc/SparcInstrAliases.td
index db4c05cf180622af9f3fbc07b4968633c3984af4..eedad25737f652a048ac77149fb6192e659db29c 100644
--- a/llvm/lib/Target/Sparc/SparcInstrAliases.td
+++ b/llvm/lib/Target/Sparc/SparcInstrAliases.td
@@ -400,9 +400,11 @@ defm : cp_cond_alias<"012",   0b1111>;
 let EmitPriority = 0 in defm : cp_cond_alias<"",      0b1000>; // same as a; gnu asm, not in manual
 
 defm : reg_cond_alias<"z",    0b001>;
+defm : reg_cond_alias<"e",    0b001>;
 defm : reg_cond_alias<"lez",  0b010>;
 defm : reg_cond_alias<"lz",   0b011>;
 defm : reg_cond_alias<"nz",   0b101>;
+defm : reg_cond_alias<"ne",   0b101>;
 defm : reg_cond_alias<"gz",   0b110>;
 defm : reg_cond_alias<"gez",  0b111>;
 
@@ -560,11 +562,16 @@ def : InstAlias<"mov $simm13, %tbr", (WRTBRri G0, simm13Op:$simm13), 0>;
 
 // End of Section A.3
 
-// or imm, reg, rd -> or reg, imm, rd
-// Nonstandard GNU extension.
-let EmitPriority = 0 in
+
+// Nonstandard GNU extensions.
+let EmitPriority = 0 in {
+  // or imm, reg, rd -> or reg, imm, rd
   def : InstAlias<"or $simm13, $rs1, $rd", (ORri IntRegs:$rd, IntRegs:$rs1, simm13Op:$simm13)>;
 
+  // addc/addx imm, reg, rd -> or reg, imm, rd
+  def : InstAlias<"addx $simm13, $rs1, $rd", (ADDCri IntRegs:$rd, IntRegs:$rs1, simm13Op:$simm13)>;
+}
+
 // wr reg_or_imm, specialreg -> wr %g0, reg_or_imm, specialreg
 // (aka: omit the first arg when it's g0. This is not in the manual, but is
 // supported by gnu and solaris as)
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td
index 4d68f93efeac17ced0a2fab5ea4076209b4841b9..cac96a13987215def294f151c176c69d60efed7f 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.td
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td
@@ -197,6 +197,16 @@ def ASITag : Operand<i32> {
   let ParserMatchClass = SparcASITagAsmOperand;
 }
 
+def SparcPrefetchTagAsmOperand : AsmOperandClass {
+  let Name = "PrefetchTag";
+  let ParserMethod = "parsePrefetchTag";
+}
+
+def PrefetchTag : Operand<i32> {
+  let PrintMethod = "printPrefetchTag";
+  let ParserMatchClass = SparcPrefetchTagAsmOperand;
+}
+
 // Branch targets have OtherVT type.
 def brtarget : Operand<OtherVT> {
   let EncoderMethod = "getBranchTargetOpValue";
@@ -1767,11 +1777,18 @@ let Predicates = [HasV9], rs1 = 0, rs2 = 0 in {
 // Section A.42 - Prefetch Data
 let Predicates = [HasV9] in {
   def PREFETCHr : F3_1<3, 0b101101,
-                   (outs), (ins (MEMrr $rs1, $rs2):$addr, shift_imm5:$rd),
+                   (outs), (ins (MEMrr $rs1, $rs2):$addr, PrefetchTag:$rd),
                    "prefetch [$addr], $rd", []>;
   def PREFETCHi : F3_2<3, 0b101101,
-                   (outs), (ins (MEMri $rs1, $simm13):$addr, shift_imm5:$rd),
+                   (outs), (ins (MEMri $rs1, $simm13):$addr, PrefetchTag:$rd),
                    "prefetch [$addr], $rd", []>;
+  def PREFETCHAr : F3_1_asi<3, 0b111101, (outs),
+                    (ins (MEMrr $rs1, $rs2):$addr, ASITag:$asi, PrefetchTag:$rd),
+                    "prefetcha [$addr] $asi, $rd", []>;
+  let Uses = [ASR3] in
+  def PREFETCHAi : F3_2<3, 0b111101, (outs),
+                    (ins (MEMri $rs1, $simm13):$addr, PrefetchTag:$rd),
+                    "prefetcha [$addr] %asi, $rd", []>;
 }
 
 
diff --git a/llvm/lib/Target/Sparc/SparcPrefetchTags.td b/llvm/lib/Target/Sparc/SparcPrefetchTags.td
new file mode 100644
index 0000000000000000000000000000000000000000..0104f47472b006a0efdc039b80534268f5c7cec5
--- /dev/null
+++ b/llvm/lib/Target/Sparc/SparcPrefetchTags.td
@@ -0,0 +1,41 @@
+//===- SparcPrefetchTags.td --------------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the symbolic operands permitted for various kinds of
+// SPARCv9 prefetches.
+//
+//===----------------------------------------------------------------------===//
+
+class PrefetchTag<string name, bits<8> op> {
+  string Name = name;
+  bits<8> Encoding = op;
+}
+
+def PrefetchTagsList : GenericTable {
+  let FilterClass = "PrefetchTag";
+  let Fields = ["Name", "Encoding"];
+
+  let PrimaryKey = [ "Encoding" ];
+  let PrimaryKeyName = "lookupPrefetchTagByEncoding";
+}
+
+def lookupPrefetchTagByName : SearchIndex {
+  let Table = PrefetchTagsList;
+  let Key = [ "Name" ];
+}
+
+def : PrefetchTag<"n_reads", 0x0>;
+def : PrefetchTag<"one_read", 0x1>;
+def : PrefetchTag<"n_writes", 0x2>;
+def : PrefetchTag<"one_write", 0x3>;
+def : PrefetchTag<"page", 0x4>;
+def : PrefetchTag<"unified", 0x11>;
+def : PrefetchTag<"n_reads_strong", 0x14>;
+def : PrefetchTag<"one_read_strong", 0x15>;
+def : PrefetchTag<"n_writes_strong", 0x16>;
+def : PrefetchTag<"one_write_strong", 0x17>;
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 8e2063121e00b17921f352a67f1f4c99858df434..f5bc584ac4e1afb0ae3b79a3136b80f27fcae389 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -178,14 +178,15 @@ static wasm::WasmLimits DefaultLimits() {
 }
 
 static MCSymbolWasm *GetOrCreateFunctionTableSymbol(MCContext &Ctx,
-                                                    const StringRef &Name) {
+                                                    const StringRef &Name,
+                                                    bool is64) {
   MCSymbolWasm *Sym = cast_or_null<MCSymbolWasm>(Ctx.lookupSymbol(Name));
   if (Sym) {
     if (!Sym->isFunctionTable())
       Ctx.reportError(SMLoc(), "symbol is not a wasm funcref table");
   } else {
     Sym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(Name));
-    Sym->setFunctionTable();
+    Sym->setFunctionTable(is64);
     // The default function table is synthesized by the linker.
     Sym->setUndefined();
   }
@@ -258,7 +259,7 @@ public:
     MCAsmParserExtension::Initialize(Parser);
 
     DefaultFunctionTable = GetOrCreateFunctionTableSymbol(
-        getContext(), "__indirect_function_table");
+        getContext(), "__indirect_function_table", is64);
     if (!STI->checkFeatures("+reference-types"))
       DefaultFunctionTable->setOmitFromLinkingSection();
   }
@@ -508,7 +509,7 @@ public:
       auto &Tok = Lexer.getTok();
       if (Tok.is(AsmToken::Identifier)) {
         auto *Sym =
-            GetOrCreateFunctionTableSymbol(getContext(), Tok.getString());
+            GetOrCreateFunctionTableSymbol(getContext(), Tok.getString(), is64);
         const auto *Val = MCSymbolRefExpr::create(Sym, getContext());
         *Op = std::make_unique<WebAssemblyOperand>(
             WebAssemblyOperand::Symbol, Tok.getLoc(), Tok.getEndLoc(),
@@ -836,6 +837,9 @@ public:
       // symbol
       auto WasmSym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(SymName));
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_TABLE);
+      if (is64) {
+        Limits.Flags |= wasm::WASM_LIMITS_FLAG_IS_64;
+      }
       wasm::WasmTableType Type = {*ElemType, Limits};
       WasmSym->setTableType(Type);
       TOut.emitTableType(WasmSym);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
index 5e7279808cce630039a13f2e9070ce6eb4779f16..c5a047ee47d730b5ee39be951946dcf4d474c927 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
@@ -108,8 +108,9 @@ MCSymbolWasm *WebAssembly::getOrCreateFunctionTableSymbol(
     if (!Sym->isFunctionTable())
       Ctx.reportError(SMLoc(), "symbol is not a wasm funcref table");
   } else {
+    bool is64 = Subtarget && Subtarget->getTargetTriple().isArch64Bit();
     Sym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(Name));
-    Sym->setFunctionTable();
+    Sym->setFunctionTable(is64);
     // The default function table is synthesized by the linker.
     Sym->setUndefined();
   }
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 0bf3294af92a884226c3a9a7df016861d42ea626..3933e82b718f2d38a9a5311d9285c860f1576ec4 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5120,6 +5120,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::x86_tileloaddt164_internal: {
       if (!Subtarget->hasAMXTILE())
         break;
+      auto *MFI =
+          CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
+      MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
       unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
                          ? X86::PTILELOADDV
                          : X86::PTILELOADDT1V;
@@ -5201,6 +5204,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
       break;
     }
     case Intrinsic::x86_tilestored64_internal: {
+      auto *MFI =
+          CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
+      MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
       unsigned Opc = X86::PTILESTOREDV;
       // _tile_stored_internal(row, col, buf, STRIDE, c)
       SDValue Base = Node->getOperand(4);
@@ -5228,6 +5234,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::x86_tilestored64: {
       if (!Subtarget->hasAMXTILE())
         break;
+      auto *MFI =
+          CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
+      MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
       unsigned Opc;
       switch (IntNo) {
       default: llvm_unreachable("Unexpected intrinsic!");
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7d30de15f84d2d9a48ed73b90a963c4c7d99d4f9..2aec14e93d082d80ff6b72f82e3f150a50364711 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -615,6 +615,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FSIN, VT, Action);
     setOperationAction(ISD::FCOS, VT, Action);
     setOperationAction(ISD::FSINCOS, VT, Action);
+    setOperationAction(ISD::FTAN, VT, Action);
     setOperationAction(ISD::FSQRT, VT, Action);
     setOperationAction(ISD::FPOW, VT, Action);
     setOperationAction(ISD::FLOG, VT, Action);
@@ -833,9 +834,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
 
     // Always expand sin/cos functions even though x87 has an instruction.
+    // clang-format off
     setOperationAction(ISD::FSIN   , MVT::f80, Expand);
     setOperationAction(ISD::FCOS   , MVT::f80, Expand);
     setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
+    setOperationAction(ISD::FTAN   , MVT::f80, Expand);
+    // clang-format on
 
     setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
     setOperationAction(ISD::FCEIL,  MVT::f80, Expand);
@@ -888,11 +892,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FNEG, MVT::f128, Custom);
     setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
 
+    // clang-format off
     setOperationAction(ISD::FSIN,         MVT::f128, LibCall);
     setOperationAction(ISD::STRICT_FSIN,  MVT::f128, LibCall);
     setOperationAction(ISD::FCOS,         MVT::f128, LibCall);
     setOperationAction(ISD::STRICT_FCOS,  MVT::f128, LibCall);
     setOperationAction(ISD::FSINCOS,      MVT::f128, LibCall);
+    setOperationAction(ISD::FTAN,         MVT::f128, LibCall);
+    setOperationAction(ISD::STRICT_FTAN,  MVT::f128, LibCall);
+    // clang-format on
     // No STRICT_FSINCOS
     setOperationAction(ISD::FSQRT,        MVT::f128, LibCall);
     setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
@@ -944,9 +952,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
                    MVT::v4f32, MVT::v8f32,  MVT::v16f32,
                    MVT::v2f64, MVT::v4f64,  MVT::v8f64 }) {
+    // clang-format off
     setOperationAction(ISD::FSIN,      VT, Expand);
     setOperationAction(ISD::FSINCOS,   VT, Expand);
     setOperationAction(ISD::FCOS,      VT, Expand);
+    setOperationAction(ISD::FTAN,      VT, Expand);
     setOperationAction(ISD::FREM,      VT, Expand);
     setOperationAction(ISD::FCOPYSIGN, VT, Expand);
     setOperationAction(ISD::FPOW,      VT, Expand);
@@ -956,6 +966,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FEXP,      VT, Expand);
     setOperationAction(ISD::FEXP2,     VT, Expand);
     setOperationAction(ISD::FEXP10,    VT, Expand);
+    // clang-format on
   }
 
   // First set operation action for all vector types to either promote
@@ -2473,7 +2484,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   // function casting to f64 and calling `fmod`.
   if (Subtarget.is32Bit() &&
       (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
-    for (ISD::NodeType Op :
+    // clang-format off
+   for (ISD::NodeType Op :
          {ISD::FCEIL,  ISD::STRICT_FCEIL,
           ISD::FCOS,   ISD::STRICT_FCOS,
           ISD::FEXP,   ISD::STRICT_FEXP,
@@ -2482,9 +2494,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
           ISD::FLOG,   ISD::STRICT_FLOG,
           ISD::FLOG10, ISD::STRICT_FLOG10,
           ISD::FPOW,   ISD::STRICT_FPOW,
-          ISD::FSIN,   ISD::STRICT_FSIN})
+          ISD::FSIN,   ISD::STRICT_FSIN,
+          ISD::FTAN,   ISD::STRICT_FTAN})
       if (isOperationExpand(Op, MVT::f32))
         setOperationAction(Op, MVT::f32, Promote);
+  // clang-format on
 
   // We have target-specific dag combine patterns for the following nodes:
   setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
@@ -26776,7 +26790,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
     case Intrinsic::swift_async_context_addr: {
       SDLoc dl(Op);
       auto &MF = DAG.getMachineFunction();
-      auto X86FI = MF.getInfo<X86MachineFunctionInfo>();
+      auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
       if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF)) {
         MF.getFrameInfo().setFrameAddressIsTaken(true);
         X86FI->setHasSwiftAsyncContext(true);
@@ -36781,7 +36795,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   }
   case TargetOpcode::PREALLOCATED_SETUP: {
     assert(Subtarget.is32Bit() && "preallocated only used in 32-bit");
-    auto MFI = MF->getInfo<X86MachineFunctionInfo>();
+    auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
     MFI->setHasPreallocatedCall(true);
     int64_t PreallocatedId = MI.getOperand(0).getImm();
     size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId);
@@ -36798,7 +36812,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit");
     int64_t PreallocatedId = MI.getOperand(1).getImm();
     int64_t ArgIdx = MI.getOperand(2).getImm();
-    auto MFI = MF->getInfo<X86MachineFunctionInfo>();
+    auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
     size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx];
     LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx
                       << ", arg offset " << ArgOffset << "\n");
@@ -36841,6 +36855,13 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     unsigned Imm = MI.getOperand(0).getImm();
     BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
     MI.eraseFromParent(); // The pseudo is gone now.
+    auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
+    MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+    return BB;
+  }
+  case X86::PTILEZEROV: {
+    auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
+    MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
     return BB;
   }
   case X86::PTILELOADD:
@@ -42431,12 +42452,10 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
     if (SimplifyDemandedBits(Op1, OriginalDemandedBits, OriginalDemandedElts,
                              Known, TLO, Depth + 1))
       return true;
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
 
     if (SimplifyDemandedBits(Op0, ~Known.Zero & OriginalDemandedBits,
                              OriginalDemandedElts, Known2, TLO, Depth + 1))
       return true;
-    assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If the RHS is a constant, see if we can simplify it.
     if (ShrinkDemandedConstant(Op, ~Known2.One & OriginalDemandedBits,
@@ -42487,7 +42506,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
                              TLO, Depth + 1))
       return true;
 
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     Known.Zero <<= ShAmt;
     Known.One <<= ShAmt;
 
@@ -42506,7 +42524,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
                              OriginalDemandedElts, Known, TLO, Depth + 1))
       return true;
 
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     Known.Zero.lshrInPlace(ShAmt);
     Known.One.lshrInPlace(ShAmt);
 
@@ -42547,7 +42564,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
                              TLO, Depth + 1))
       return true;
 
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     Known.Zero.lshrInPlace(ShAmt);
     Known.One.lshrInPlace(ShAmt);
 
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index c47bee070e04fa805d1b40d8c811054d9878c1d7..99deacc811a170b87d6c7c783c4d231814bec0d7 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -74,7 +74,7 @@ let SchedRW = [WriteSystem] in {
                                             GR16:$src2, opaquemem:$src3,
                                             TILE:$src4), []>;
     let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
-        canFoldAsLoad = 1 in
+        canFoldAsLoad = 1, usesCustomInserter = 1 in
       def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
                                 [(set TILE:$dst, (int_x86_tilezero_internal
                                   GR16:$src1, GR16:$src2))]>;
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 270dd32c7235a2cf039776703790776a06e51f01..6fb6e1633b0c9c23b39f7aed088eb8bf0f704285 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1125,7 +1125,9 @@ multiclass ATOMIC_LOAD_FP_BINOP_MI<string Name, SDNode op> {
             Requires<[HasAVX512]>;
 }
 defm : ATOMIC_LOAD_FP_BINOP_MI<"ADD", fadd>;
-// FIXME: Add fsub, fmul, fdiv, ...
+defm : ATOMIC_LOAD_FP_BINOP_MI<"SUB", fsub>;
+defm : ATOMIC_LOAD_FP_BINOP_MI<"MUL", fmul>;
+defm : ATOMIC_LOAD_FP_BINOP_MI<"DIV", fdiv>;
 
 multiclass RELEASE_UNOP<string Name, dag dag8, dag dag16, dag dag32,
                         dag dag64> {
@@ -1570,21 +1572,21 @@ let Predicates = [HasNDD] in {
 }
 
 // Depositing value to 8/16 bit subreg:
-def : Pat<(or (and GR64:$dst, -256), 
+def : Pat<(or (and GR64:$dst, -256),
               (i64 (zextloadi8 addr:$src))),
-          (INSERT_SUBREG (i64 (COPY $dst)), (MOV8rm  i8mem:$src), sub_8bit)>; 
+          (INSERT_SUBREG (i64 (COPY $dst)), (MOV8rm  i8mem:$src), sub_8bit)>;
 
-def : Pat<(or (and GR32:$dst, -256), 
+def : Pat<(or (and GR32:$dst, -256),
               (i32 (zextloadi8 addr:$src))),
-          (INSERT_SUBREG (i32 (COPY $dst)), (MOV8rm  i8mem:$src), sub_8bit)>; 
+          (INSERT_SUBREG (i32 (COPY $dst)), (MOV8rm  i8mem:$src), sub_8bit)>;
 
-def : Pat<(or (and GR64:$dst, -65536), 
+def : Pat<(or (and GR64:$dst, -65536),
               (i64 (zextloadi16 addr:$src))),
           (INSERT_SUBREG (i64 (COPY $dst)), (MOV16rm  i16mem:$src), sub_16bit)>;
 
-def : Pat<(or (and GR32:$dst, -65536), 
+def : Pat<(or (and GR32:$dst, -65536),
               (i32 (zextloadi16 addr:$src))),
-          (INSERT_SUBREG (i32 (COPY $dst)), (MOV16rm  i16mem:$src), sub_16bit)>; 
+          (INSERT_SUBREG (i32 (COPY $dst)), (MOV16rm  i16mem:$src), sub_16bit)>;
 
 // To avoid needing to materialize an immediate in a register, use a 32-bit and
 // with implicit zero-extension instead of a 64-bit and if the immediate has at
diff --git a/llvm/lib/Target/X86/X86InstrMisc.td b/llvm/lib/Target/X86/X86InstrMisc.td
index 496a7e6b2943629a9b0d388c88f87b8bcedf510a..c4da0e50a1dd8b7bb201e4f9259d1bf0c83d918b 100644
--- a/llvm/lib/Target/X86/X86InstrMisc.td
+++ b/llvm/lib/Target/X86/X86InstrMisc.td
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This file defining the misc X86 instructions.
-// 
+//
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index b69058787a4e25a7114b59f7cfae90f2a873ca32..079ac983a8a01d8bafcf5fc7fe7771dad7ed937e 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -92,6 +92,14 @@ static bool isAMXIntrinsic(Value *I) {
   return false;
 }
 
+static bool containsAMXCode(Function &F) {
+  for (BasicBlock &BB : F)
+    for (Instruction &I : BB)
+      if (I.getType()->isX86_AMXTy())
+        return true;
+  return false;
+}
+
 static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
                                            Type *Ty) {
   Function &F = *BB->getParent();
@@ -1230,6 +1238,14 @@ public:
   }
 
   bool runOnFunction(Function &F) override {
+    // Performance optimization: most code doesn't use AMX, so return early if
+    // there are no instructions that produce AMX values. This is sufficient, as
+    // AMX arguments and constants are not allowed -- so any producer of an AMX
+    // value must be an instruction.
+    // TODO: find a cheaper way for this, without looking at all instructions.
+    if (!containsAMXCode(F))
+      return false;
+
     bool C = false;
     TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
     TargetLibraryInfo *TLI =
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index f27676a27e86cd0ec9f90c3a346ad50ec20819b5..613722b398f4469201332537c77d206c176dca7d 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -19,6 +19,7 @@
 #include "X86.h"
 #include "X86InstrBuilder.h"
 #include "X86InstrInfo.h"
+#include "X86MachineFunctionInfo.h"
 #include "X86Subtarget.h"
 #include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -71,6 +72,10 @@ FunctionPass *llvm::createX86LowerTileCopyPass() {
 }
 
 bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
+  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
+  if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+    return false;
+
   const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
   const X86InstrInfo *TII = ST.getInstrInfo();
   const TargetRegisterInfo *TRI = ST.getRegisterInfo();
@@ -81,26 +86,8 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {
-    // There won't be a tile copy if neither tile register live in nor live out.
-    bool HasTileCopy = false;
-    for (const auto &LI : MBB.liveins()) {
-      if (TILERegs.test(LI.PhysReg)) {
-        HasTileCopy = true;
-        break;
-      }
-    }
     LiveRegUnits UsedRegs(*TRI);
     UsedRegs.addLiveOuts(MBB);
-    if (!HasTileCopy) {
-      for (auto RegT : TILERegs.set_bits()) {
-        if (UsedRegs.available(RegT)) {
-          HasTileCopy = true;
-          break;
-        }
-      }
-    }
-    if (!HasTileCopy)
-      continue;
     for (MachineInstr &MI : llvm::make_early_inc_range(reverse(MBB))) {
       UsedRegs.stepBackward(MI);
       if (!MI.isCopy())
diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
index f6e853270e073bde3a32b1c230f0e8f2066af4fb..8aaa49945f9d4478440cc0edc7be1a746b73a315 100644
--- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
@@ -21,6 +21,8 @@
 
 namespace llvm {
 
+enum AMXProgModelEnum { None = 0, DirectReg = 1, ManagedRA = 2 };
+
 /// X86MachineFunctionInfo - This class is derived from MachineFunction and
 /// contains private X86 target-specific information for each MachineFunction.
 class X86MachineFunctionInfo : public MachineFunctionInfo {
@@ -96,6 +98,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   /// used to address arguments in a function using a base pointer.
   int SEHFramePtrSaveIndex = 0;
 
+  /// The AMX programing model used in the function.
+  AMXProgModelEnum AMXProgModel = AMXProgModelEnum::None;
+
   /// True if this function has a subset of CSRs that is handled explicitly via
   /// copies.
   bool IsSplitCSR = false;
@@ -219,6 +224,13 @@ public:
   int getSEHFramePtrSaveIndex() const { return SEHFramePtrSaveIndex; }
   void setSEHFramePtrSaveIndex(int Index) { SEHFramePtrSaveIndex = Index; }
 
+  AMXProgModelEnum getAMXProgModel() const { return AMXProgModel; }
+  void setAMXProgModel(AMXProgModelEnum Model) {
+    assert((AMXProgModel == AMXProgModelEnum::None || AMXProgModel == Model) &&
+           "mixed model is not supported");
+    AMXProgModel = Model;
+  }
+
   SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
     return ForwardedMustTailRegParms;
   }
diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 2d296771b1c07d7b4b6d3119d3e8910c9ece3856..186d4d84c25104c8e482ce9e6601729620222976 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -620,11 +620,11 @@ def : WriteRes<WriteNop, []>;
 // Horizontal add/sub  instructions.
 ////////////////////////////////////////////////////////////////////////////////
 
-defm : ICXWriteResPair<WriteFHAdd,  [ICXPort5,ICXPort015], 6, [2,1], 3, 6>;
-defm : ICXWriteResPair<WriteFHAddY, [ICXPort5,ICXPort015], 6, [2,1], 3, 7>;
+defm : ICXWriteResPair<WriteFHAdd,  [ICXPort5,ICXPort01], 6, [2,1], 3, 6>;
+defm : ICXWriteResPair<WriteFHAddY, [ICXPort5,ICXPort01], 6, [2,1], 3, 7>;
 defm : ICXWriteResPair<WritePHAdd,  [ICXPort5,ICXPort05],  3, [2,1], 3, 5>;
-defm : ICXWriteResPair<WritePHAddX, [ICXPort5,ICXPort015], 3, [2,1], 3, 6>;
-defm : ICXWriteResPair<WritePHAddY, [ICXPort5,ICXPort015], 3, [2,1], 3, 7>;
+defm : ICXWriteResPair<WritePHAddX, [ICXPort15,ICXPort015], 3, [2,1], 3, 6>;
+defm : ICXWriteResPair<WritePHAddY, [ICXPort15,ICXPort015], 3, [2,1], 3, 7>;
 
 // Remaining instrs.
 
@@ -886,7 +886,7 @@ def ICXWriteResGroup37 : SchedWriteRes<[ICXPort0,ICXPort5]> {
 }
 def: InstRW<[ICXWriteResGroup37], (instregex "MMX_PH(ADD|SUB)SWrr")>;
 
-def ICXWriteResGroup38 : SchedWriteRes<[ICXPort5,ICXPort01]> {
+def ICXWriteResGroup38 : SchedWriteRes<[ICXPort15,ICXPort01]> {
   let Latency = 3;
   let NumMicroOps = 3;
   let ReleaseAtCycles = [2,1];
@@ -1739,13 +1739,13 @@ def ICXWriteResGroup137 : SchedWriteRes<[ICXPort23,ICXPort01]> {
 def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm",
                                               "(V?)CVTPS2PDrm")>;
 
-def ICXWriteResGroup143 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> {
+def ICXWriteResGroup143 : SchedWriteRes<[ICXPort15,ICXPort01,ICXPort23]> {
   let Latency = 9;
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
 }
-def: InstRW<[ICXWriteResGroup143], (instregex "(V?)PHADDSWrm",
-                                              "(V?)PHSUBSWrm")>;
+def: InstRW<[ICXWriteResGroup143], (instrs PHADDSWrm, VPHADDSWrm,
+                                           PHSUBSWrm, VPHSUBSWrm)>;
 
 def ICXWriteResGroup146 : SchedWriteRes<[ICXPort1,ICXPort6,ICXPort23,ICXPort0156]> {
   let Latency = 9;
@@ -1842,7 +1842,7 @@ def: InstRW<[ICXWriteResGroup151], (instregex "VEXPANDPDZ128rm(b?)",
                                               "VPEXPANDDZ128rm(b?)",
                                               "VPEXPANDQZ128rm(b?)")>;
 
-def ICXWriteResGroup154 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> {
+def ICXWriteResGroup154 : SchedWriteRes<[ICXPort15,ICXPort01,ICXPort23]> {
   let Latency = 10;
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index a7dff0ecbcd96f2b18b68088f3eb8db8f31cea38..4fded44085e89737e3f89be0fa7032a16cbaaf23 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -615,8 +615,8 @@ def : WriteRes<WriteNop, []>;
 // Horizontal add/sub  instructions.
 ////////////////////////////////////////////////////////////////////////////////
 
-defm : SKXWriteResPair<WriteFHAdd,  [SKXPort5,SKXPort015], 6, [2,1], 3, 6>;
-defm : SKXWriteResPair<WriteFHAddY, [SKXPort5,SKXPort015], 6, [2,1], 3, 7>;
+defm : SKXWriteResPair<WriteFHAdd,  [SKXPort5,SKXPort01], 6, [2,1], 3, 6>;
+defm : SKXWriteResPair<WriteFHAddY, [SKXPort5,SKXPort01], 6, [2,1], 3, 7>;
 defm : SKXWriteResPair<WritePHAdd,  [SKXPort5,SKXPort05],  3, [2,1], 3, 5>;
 defm : SKXWriteResPair<WritePHAddX, [SKXPort5,SKXPort015], 3, [2,1], 3, 6>;
 defm : SKXWriteResPair<WritePHAddY, [SKXPort5,SKXPort015], 3, [2,1], 3, 7>;
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index ca356ec82bf1f9330f7f7b1e3f95ea5209ae6656..d1cc306790522dd078775abc7977d5fafba0a2c4 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -122,6 +122,14 @@ AArch64::parseArchExtension(StringRef ArchExt) {
   return {};
 }
 
+std::optional<AArch64::ExtensionInfo>
+AArch64::targetFeatureToExtension(StringRef TargetFeature) {
+  for (const auto &E : Extensions)
+    if (TargetFeature == E.Feature)
+      return E;
+  return {};
+}
+
 std::optional<AArch64::CpuInfo> AArch64::parseCpu(StringRef Name) {
   // Resolve aliases first.
   Name = resolveCPUAlias(Name);
@@ -213,21 +221,6 @@ void AArch64::ExtensionSet::disable(ArchExtKind E) {
       disable(Dep.Later);
 }
 
-void AArch64::ExtensionSet::toLLVMFeatureList(
-    std::vector<StringRef> &Features) const {
-  if (BaseArch && !BaseArch->ArchFeature.empty())
-    Features.push_back(BaseArch->ArchFeature);
-
-  for (const auto &E : Extensions) {
-    if (E.Feature.empty() || !Touched.test(E.ID))
-      continue;
-    if (Enabled.test(E.ID))
-      Features.push_back(E.Feature);
-    else
-      Features.push_back(E.NegFeature);
-  }
-}
-
 void AArch64::ExtensionSet::addCPUDefaults(const CpuInfo &CPU) {
   LLVM_DEBUG(llvm::dbgs() << "addCPUDefaults(" << CPU.Name << ")\n");
   BaseArch = &CPU.Arch;
@@ -247,11 +240,18 @@ void AArch64::ExtensionSet::addArchDefaults(const ArchInfo &Arch) {
       enable(E.ID);
 }
 
-bool AArch64::ExtensionSet::parseModifier(StringRef Modifier) {
+bool AArch64::ExtensionSet::parseModifier(StringRef Modifier,
+                                          const bool AllowNoDashForm) {
   LLVM_DEBUG(llvm::dbgs() << "parseModifier(" << Modifier << ")\n");
 
-  bool IsNegated = Modifier.starts_with("no");
-  StringRef ArchExt = IsNegated ? Modifier.drop_front(2) : Modifier;
+  size_t NChars = 0;
+  // The "no-feat" form is allowed in the target attribute but nowhere else.
+  if (AllowNoDashForm && Modifier.starts_with("no-"))
+    NChars = 3;
+  else if (Modifier.starts_with("no"))
+    NChars = 2;
+  bool IsNegated = NChars != 0;
+  StringRef ArchExt = Modifier.drop_front(NChars);
 
   if (auto AE = parseArchExtension(ArchExt)) {
     if (AE->Feature.empty() || AE->NegFeature.empty())
@@ -265,6 +265,21 @@ bool AArch64::ExtensionSet::parseModifier(StringRef Modifier) {
   return false;
 }
 
+void AArch64::ExtensionSet::reconstructFromParsedFeatures(
+    const std::vector<std::string> &Features) {
+  assert(Touched.none() && "Bitset already initialized");
+  for (auto &F : Features) {
+    bool IsNegated = F[0] == '-';
+    if (auto AE = targetFeatureToExtension(F)) {
+      Touched.set(AE->ID);
+      if (IsNegated)
+        Enabled.reset(AE->ID);
+      else
+        Enabled.set(AE->ID);
+    }
+  }
+}
+
 const AArch64::ExtensionInfo &
 AArch64::getExtensionByID(AArch64::ArchExtKind ExtID) {
   return lookupExtensionByID(ExtID);
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 68155acd9e5bcd350b25be373576f5d10f824b66..369287e2d2037c8fce2b207a533c89060710fafa 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -213,6 +213,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
         .Case("0xd23", "cortex-m85")
         .Case("0xc18", "cortex-r8")
         .Case("0xd13", "cortex-r52")
+        .Case("0xd16", "cortex-r52plus")
         .Case("0xd15", "cortex-r82")
         .Case("0xd14", "cortex-r82ae")
         .Case("0xd02", "cortex-a34")
@@ -302,6 +303,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
         .Case("0x805", "cortex-a76") // Kryo 4xx/5xx Silver
         .Case("0xc00", "falkor")
         .Case("0xc01", "saphira")
+        .Case("0x001", "oryon-1")
         .Default("generic");
   if (Implementer == "0x53") { // Samsung Electronics Co., Ltd.
     // The Exynos chips have a convoluted ID scheme that doesn't seem to follow
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 7464237d26d4815fe4ab6c9e40f08c56f40484e8..00df92e0aadedaa9d48eda4cab10d3dbf39e6ea9 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -124,6 +124,7 @@ constexpr GPUInfo AMDGCNGPUs[] = {
     {{"gfx1103"},   {"gfx1103"}, GK_GFX1103, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1150"},   {"gfx1150"}, GK_GFX1150, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1151"},   {"gfx1151"}, GK_GFX1151, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+    {{"gfx1152"},   {"gfx1152"}, GK_GFX1152, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1200"},   {"gfx1200"}, GK_GFX1200, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1201"},   {"gfx1201"}, GK_GFX1201, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
 
@@ -275,6 +276,7 @@ AMDGPU::IsaVersion AMDGPU::getIsaVersion(StringRef GPU) {
   case GK_GFX1103: return {11, 0, 3};
   case GK_GFX1150: return {11, 5, 0};
   case GK_GFX1151: return {11, 5, 1};
+  case GK_GFX1152: return {11, 5, 2};
   case GK_GFX1200: return {12, 0, 0};
   case GK_GFX1201: return {12, 0, 1};
 
@@ -313,7 +315,47 @@ StringRef AMDGPU::getCanonicalArchName(const Triple &T, StringRef Arch) {
 void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
                                   StringMap<bool> &Features) {
   // XXX - What does the member GPU mean if device name string passed here?
-  if (T.isAMDGCN()) {
+  if (T.isSPIRV() && T.getOS() == Triple::OSType::AMDHSA) {
+    // AMDGCN SPIRV must support the union of all AMDGCN features.
+    Features["atomic-ds-pk-add-16-insts"] = true;
+    Features["atomic-flat-pk-add-16-insts"] = true;
+    Features["atomic-buffer-global-pk-add-f16-insts"] = true;
+    Features["atomic-global-pk-add-bf16-inst"] = true;
+    Features["atomic-fadd-rtn-insts"] = true;
+    Features["ci-insts"] = true;
+    Features["dot1-insts"] = true;
+    Features["dot2-insts"] = true;
+    Features["dot3-insts"] = true;
+    Features["dot4-insts"] = true;
+    Features["dot5-insts"] = true;
+    Features["dot7-insts"] = true;
+    Features["dot8-insts"] = true;
+    Features["dot9-insts"] = true;
+    Features["dot10-insts"] = true;
+    Features["dot11-insts"] = true;
+    Features["dl-insts"] = true;
+    Features["16-bit-insts"] = true;
+    Features["dpp"] = true;
+    Features["gfx8-insts"] = true;
+    Features["gfx9-insts"] = true;
+    Features["gfx90a-insts"] = true;
+    Features["gfx940-insts"] = true;
+    Features["gfx10-insts"] = true;
+    Features["gfx10-3-insts"] = true;
+    Features["gfx11-insts"] = true;
+    Features["gfx12-insts"] = true;
+    Features["image-insts"] = true;
+    Features["fp8-conversion-insts"] = true;
+    Features["s-memrealtime"] = true;
+    Features["s-memtime-inst"] = true;
+    Features["gws"] = true;
+    Features["fp8-insts"] = true;
+    Features["fp8-conversion-insts"] = true;
+    Features["atomic-ds-pk-add-16-insts"] = true;
+    Features["mai-insts"] = true;
+    Features["wavefrontsize32"] = true;
+    Features["wavefrontsize64"] = true;
+  } else if (T.isAMDGCN()) {
     switch (parseArchAMDGCN(GPU)) {
     case GK_GFX1201:
     case GK_GFX1200:
@@ -341,6 +383,7 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
       Features["image-insts"] = true;
       Features["fp8-conversion-insts"] = true;
       break;
+    case GK_GFX1152:
     case GK_GFX1151:
     case GK_GFX1150:
     case GK_GFX1103:
@@ -542,6 +585,7 @@ static bool isWave32Capable(StringRef GPU, const Triple &T) {
     switch (parseArchAMDGCN(GPU)) {
     case GK_GFX1201:
     case GK_GFX1200:
+    case GK_GFX1152:
     case GK_GFX1151:
     case GK_GFX1150:
     case GK_GFX1103:
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 5fbdbc3a014f9aae53d0be809a2fc0a4cbaf297a..92a9697720efd4e4e1eb94e2c39d3727b2379247 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_component_library(LLVMipo
   DeadArgumentElimination.cpp
   ElimAvailExtern.cpp
   EmbedBitcodePass.cpp
+  ExpandVariadics.cpp
   ExtractGV.cpp
   ForceFunctionAttrs.cpp
   FunctionAttrs.cpp
diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d340bc041ccdad5cb73ff7859f9a4835c651a1a8
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
@@ -0,0 +1,1012 @@
+//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is an optimization pass for variadic functions. If called from codegen,
+// it can serve as the implementation of variadic functions for a given target.
+//
+// The strategy is to turn the ... part of a variadic function into a va_list
+// and fix up the call sites. The majority of the pass is target independent.
+// The exceptions are the va_list type itself and the rules for where to store
+// variables in memory such that va_arg can iterate over them given a va_list.
+//
+// The majority of the plumbing is splitting the variadic function into a
+// single basic block that packs the variadic arguments into a va_list and
+// a second function that does the work of the original. That packing is
+// exactly what is done by va_start. Further, the transform from ... to va_list
+// replaced va_start with an operation to copy a va_list from the new argument,
+// which is exactly a va_copy. This is useful for reducing target-dependence.
+//
+// A va_list instance is a forward iterator, where the primary operation va_arg
+// is dereference-then-increment. This interface forces significant convergent
+// evolution between target specific implementations. The variation in runtime
+// data layout is limited to that representable by the iterator, parameterised
+// by the type passed to the va_arg instruction.
+//
+// Therefore the majority of the target specific subtlety is packing arguments
+// into a stack allocated buffer such that a va_list can be initialised with it
+// and the va_arg expansion for the target will find the arguments at runtime.
+//
+// The aggregate effect is to unblock other transforms, most critically the
+// general purpose inliner. Known calls to variadic functions become zero cost.
+//
+// Consistency with clang is primarily tested by emitting va_arg using clang
+// then expanding the variadic functions using this pass, followed by trying
+// to constant fold the functions to no-ops.
+//
+// Target specific behaviour is tested in IR - mainly checking that values are
+// put into positions in call frames that make sense for that particular target.
+//
+// There is one "clever" invariant in use. va_start intrinsics that are not
+// within a varidic functions are an error in the IR verifier. When this
+// transform moves blocks from a variadic function into a fixed arity one, it
+// moves va_start intrinsics along with everything else. That means that the
+// va_start intrinsics that need to be rewritten to use the trailing argument
+// are exactly those that are in non-variadic functions so no further state
+// is needed to distinguish those that need to be rewritten.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/ExpandVariadics.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+#define DEBUG_TYPE "expand-variadics"
+
+using namespace llvm;
+
+namespace {
+
+cl::opt<ExpandVariadicsMode> ExpandVariadicsModeOption(
+    DEBUG_TYPE "-override", cl::desc("Override the behaviour of " DEBUG_TYPE),
+    cl::init(ExpandVariadicsMode::Unspecified),
+    cl::values(clEnumValN(ExpandVariadicsMode::Unspecified, "unspecified",
+                          "Use the implementation defaults"),
+               clEnumValN(ExpandVariadicsMode::Disable, "disable",
+                          "Disable the pass entirely"),
+               clEnumValN(ExpandVariadicsMode::Optimize, "optimize",
+                          "Optimise without changing ABI"),
+               clEnumValN(ExpandVariadicsMode::Lowering, "lowering",
+                          "Change variadic calling convention")));
+
+bool commandLineOverride() {
+  return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified;
+}
+
+// Instances of this class encapsulate the target-dependant behaviour as a
+// function of triple. Implementing a new ABI is adding a case to the switch
+// in create(llvm::Triple) at the end of this file.
+// This class may end up instantiated in TargetMachine instances, keeping it
+// here for now until enough targets are implemented for the API to evolve.
+class VariadicABIInfo {
+protected:
+  VariadicABIInfo() = default;
+
+public:
+  static std::unique_ptr<VariadicABIInfo> create(const Triple &T);
+
+  // Allow overriding whether the pass runs on a per-target basis
+  virtual bool enableForTarget() = 0;
+
+  // Whether a valist instance is passed by value or by address
+  // I.e. does it need to be alloca'ed and stored into, or can
+  // it be passed directly in a SSA register
+  virtual bool vaListPassedInSSARegister() = 0;
+
+  // The type of a va_list iterator object
+  virtual Type *vaListType(LLVMContext &Ctx) = 0;
+
+  // The type of a va_list as a function argument as lowered by C
+  virtual Type *vaListParameterType(Module &M) = 0;
+
+  // Initialize an allocated va_list object to point to an already
+  // initialized contiguous memory region.
+  // Return the value to pass as the va_list argument
+  virtual Value *initializeVaList(Module &M, LLVMContext &Ctx,
+                                  IRBuilder<> &Builder, AllocaInst *VaList,
+                                  Value *Buffer) = 0;
+
+  struct VAArgSlotInfo {
+    Align DataAlign; // With respect to the call frame
+    bool Indirect;   // Passed via a pointer
+  };
+  virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0;
+
+  // Targets implemented so far all have the same trivial lowering for these
+  bool vaEndIsNop() { return true; }
+  bool vaCopyIsMemcpy() { return true; }
+
+  virtual ~VariadicABIInfo() = default;
+};
+
+// Module implements getFunction() which returns nullptr on missing declaration
+// and getOrInsertFunction which creates one when absent. Intrinsics.h only
+// implements getDeclaration which creates one when missing. Checking whether
+// an intrinsic exists thus inserts it in the module and it then needs to be
+// deleted again to clean up.
+// The right name for the two functions on intrinsics would match Module::,
+// but doing that in a single change would introduce nullptr dereferences
+// where currently there are none. The minimal collateral damage approach
+// would split the change over a release to help downstream branches. As it
+// is unclear what approach will be preferred, implementing the trivial
+// function here in the meantime to decouple from that discussion.
+Function *getPreexistingDeclaration(Module *M, Intrinsic::ID Id,
+                                    ArrayRef<Type *> Tys = {}) {
+  auto *FT = Intrinsic::getType(M->getContext(), Id, Tys);
+  return M->getFunction(Tys.empty() ? Intrinsic::getName(Id)
+                                    : Intrinsic::getName(Id, Tys, M, FT));
+}
+
+class ExpandVariadics : public ModulePass {
+
+  // The pass construction sets the default to optimize when called from middle
+  // end and lowering when called from the backend. The command line variable
+  // overrides that. This is useful for testing and debugging. It also allows
+  // building an applications with variadic functions wholly removed if one
+  // has sufficient control over the dependencies, e.g. a statically linked
+  // clang that has no variadic function calls remaining in the binary.
+
+public:
+  static char ID;
+  const ExpandVariadicsMode Mode;
+  std::unique_ptr<VariadicABIInfo> ABI;
+
+  ExpandVariadics(ExpandVariadicsMode Mode)
+      : ModulePass(ID),
+        Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {}
+
+  StringRef getPassName() const override { return "Expand variadic functions"; }
+
+  bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; }
+
+  bool runOnModule(Module &M) override;
+
+  bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F);
+
+  Function *replaceAllUsesWithNewDeclaration(Module &M,
+                                             Function *OriginalFunction);
+
+  Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
+                                        Function *OriginalFunction);
+
+  Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
+                                  Function *VariadicWrapper,
+                                  Function *FixedArityReplacement);
+
+  bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *,
+                  Function *NF);
+
+  // The intrinsic functions va_copy and va_end are removed unconditionally.
+  // They correspond to a memcpy and a no-op on all implemented targets.
+  // The va_start intrinsic is removed from basic blocks that were not created
+  // by this pass, some may remain if needed to maintain the external ABI.
+
+  template <Intrinsic::ID ID, typename InstructionType>
+  bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder,
+                            PointerType *IntrinsicArgType) {
+    bool Changed = false;
+    const DataLayout &DL = M.getDataLayout();
+    if (Function *Intrinsic =
+            getPreexistingDeclaration(&M, ID, {IntrinsicArgType})) {
+      for (User *U : make_early_inc_range(Intrinsic->users()))
+        if (auto *I = dyn_cast<InstructionType>(U))
+          Changed |= expandVAIntrinsicCall(Builder, DL, I);
+
+      if (Intrinsic->use_empty())
+        Intrinsic->eraseFromParent();
+    }
+    return Changed;
+  }
+
+  bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder,
+                                           unsigned Addrspace) {
+    auto &Ctx = M.getContext();
+    PointerType *IntrinsicArgType = PointerType::get(Ctx, Addrspace);
+    bool Changed = false;
+
+    // expand vastart before vacopy as vastart may introduce a vacopy
+    Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(
+        M, Builder, IntrinsicArgType);
+    Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(
+        M, Builder, IntrinsicArgType);
+    Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(
+        M, Builder, IntrinsicArgType);
+    return Changed;
+  }
+
+  bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
+                             VAStartInst *Inst);
+
+  bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
+                             VAEndInst *Inst);
+
+  bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
+                             VACopyInst *Inst);
+
+  FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) {
+    // The type of "FTy" with the ... removed and a va_list appended
+    SmallVector<Type *> ArgTypes(FTy->param_begin(), FTy->param_end());
+    ArgTypes.push_back(ABI->vaListParameterType(M));
+    return FunctionType::get(FTy->getReturnType(), ArgTypes,
+                             /*IsVarArgs=*/false);
+  }
+
+  static ConstantInt *sizeOfAlloca(LLVMContext &Ctx, const DataLayout &DL,
+                                   AllocaInst *Alloced) {
+    std::optional<TypeSize> AllocaTypeSize = Alloced->getAllocationSize(DL);
+    uint64_t AsInt = AllocaTypeSize ? AllocaTypeSize->getFixedValue() : 0;
+    return ConstantInt::get(Type::getInt64Ty(Ctx), AsInt);
+  }
+
+  bool expansionApplicableToFunction(Module &M, Function *F) {
+    if (F->isIntrinsic() || !F->isVarArg() ||
+        F->hasFnAttribute(Attribute::Naked))
+      return false;
+
+    if (F->getCallingConv() != CallingConv::C)
+      return false;
+
+    if (rewriteABI())
+      return true;
+
+    if (!F->hasExactDefinition())
+      return false;
+
+    return true;
+  }
+
+  bool expansionApplicableToFunctionCall(CallBase *CB) {
+    if (CallInst *CI = dyn_cast<CallInst>(CB)) {
+      if (CI->isMustTailCall()) {
+        // Cannot expand musttail calls
+        return false;
+      }
+
+      if (CI->getCallingConv() != CallingConv::C)
+        return false;
+
+      return true;
+    }
+
+    if (isa<InvokeInst>(CB)) {
+      // Invoke not implemented in initial implementation of pass
+      return false;
+    }
+
+    // Other unimplemented derivative of CallBase
+    return false;
+  }
+
+  class ExpandedCallFrame {
+    // Helper for constructing an alloca instance containing the arguments bound
+    // to the variadic ... parameter, rearranged to allow indexing through a
+    // va_list iterator
+    enum { N = 4 };
+    SmallVector<Type *, N> FieldTypes;
+    enum Tag { Store, Memcpy, Padding };
+    SmallVector<std::tuple<Value *, uint64_t, Tag>, N> Source;
+
+    template <Tag tag> void append(Type *FieldType, Value *V, uint64_t Bytes) {
+      FieldTypes.push_back(FieldType);
+      Source.push_back({V, Bytes, tag});
+    }
+
+  public:
+    void store(LLVMContext &Ctx, Type *T, Value *V) { append<Store>(T, V, 0); }
+
+    void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) {
+      append<Memcpy>(T, V, Bytes);
+    }
+
+    void padding(LLVMContext &Ctx, uint64_t By) {
+      append<Padding>(ArrayType::get(Type::getInt8Ty(Ctx), By), nullptr, 0);
+    }
+
+    size_t size() const { return FieldTypes.size(); }
+    bool empty() const { return FieldTypes.empty(); }
+
+    StructType *asStruct(LLVMContext &Ctx, StringRef Name) {
+      const bool IsPacked = true;
+      return StructType::create(Ctx, FieldTypes,
+                                (Twine(Name) + ".vararg").str(), IsPacked);
+    }
+
+    void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder,
+                                AllocaInst *Alloced) {
+
+      StructType *VarargsTy = cast<StructType>(Alloced->getAllocatedType());
+
+      for (size_t I = 0; I < size(); I++) {
+
+        auto [V, bytes, tag] = Source[I];
+
+        if (tag == Padding) {
+          assert(V == nullptr);
+          continue;
+        }
+
+        auto Dst = Builder.CreateStructGEP(VarargsTy, Alloced, I);
+
+        assert(V != nullptr);
+
+        if (tag == Store)
+          Builder.CreateStore(V, Dst);
+
+        if (tag == Memcpy)
+          Builder.CreateMemCpy(Dst, {}, V, {}, bytes);
+      }
+    }
+  };
+};
+
+bool ExpandVariadics::runOnModule(Module &M) {
+  bool Changed = false;
+  if (Mode == ExpandVariadicsMode::Disable)
+    return Changed;
+
+  Triple TT(M.getTargetTriple());
+  ABI = VariadicABIInfo::create(TT);
+  if (!ABI)
+    return Changed;
+
+  if (!ABI->enableForTarget())
+    return Changed;
+
+  auto &Ctx = M.getContext();
+  const DataLayout &DL = M.getDataLayout();
+  IRBuilder<> Builder(Ctx);
+
+  // Lowering needs to run on all functions exactly once.
+  // Optimize could run on functions containing va_start exactly once.
+  for (Function &F : make_early_inc_range(M))
+    Changed |= runOnFunction(M, Builder, &F);
+
+  // After runOnFunction, all known calls to known variadic functions have been
+  // replaced. va_start intrinsics are presently (and invalidly!) only present
+  // in functions that used to be variadic and have now been replaced to take a
+  // va_list instead. If lowering as opposed to optimising, calls to unknown
+  // variadic functions have also been replaced.
+
+  {
+    // 0 and AllocaAddrSpace are sufficient for the targets implemented so far
+    unsigned Addrspace = 0;
+    Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
+
+    Addrspace = DL.getAllocaAddrSpace();
+    if (Addrspace != 0)
+      Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
+  }
+
+  if (Mode != ExpandVariadicsMode::Lowering)
+    return Changed;
+
+  for (Function &F : make_early_inc_range(M)) {
+    if (F.isDeclaration())
+      continue;
+
+    // Now need to track down indirect calls. Can't find those
+    // by walking uses of variadic functions, need to crawl the instruction
+    // stream. Fortunately this is only necessary for the ABI rewrite case.
+    for (BasicBlock &BB : F) {
+      for (Instruction &I : make_early_inc_range(BB)) {
+        if (CallBase *CB = dyn_cast<CallBase>(&I)) {
+          if (CB->isIndirectCall()) {
+            FunctionType *FTy = CB->getFunctionType();
+            if (FTy->isVarArg())
+              Changed |= expandCall(M, Builder, CB, FTy, 0);
+          }
+        }
+      }
+    }
+  }
+
+  return Changed;
+}
+
+bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder,
+                                    Function *OriginalFunction) {
+  bool Changed = false;
+
+  if (!expansionApplicableToFunction(M, OriginalFunction))
+    return Changed;
+
+  [[maybe_unused]] const bool OriginalFunctionIsDeclaration =
+      OriginalFunction->isDeclaration();
+  assert(rewriteABI() || !OriginalFunctionIsDeclaration);
+
+  // Declare a new function and redirect every use to that new function
+  Function *VariadicWrapper =
+      replaceAllUsesWithNewDeclaration(M, OriginalFunction);
+  assert(VariadicWrapper->isDeclaration());
+  assert(OriginalFunction->use_empty());
+
+  // Create a new function taking va_list containing the implementation of the
+  // original
+  Function *FixedArityReplacement =
+      deriveFixedArityReplacement(M, Builder, OriginalFunction);
+  assert(OriginalFunction->isDeclaration());
+  assert(FixedArityReplacement->isDeclaration() ==
+         OriginalFunctionIsDeclaration);
+  assert(VariadicWrapper->isDeclaration());
+
+  // Create a single block forwarding wrapper that turns a ... into a va_list
+  [[maybe_unused]] Function *VariadicWrapperDefine =
+      defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);
+  assert(VariadicWrapperDefine == VariadicWrapper);
+  assert(!VariadicWrapper->isDeclaration());
+
+  // We now have:
+  // 1. the original function, now as a declaration with no uses
+  // 2. a variadic function that unconditionally calls a fixed arity replacement
+  // 3. a fixed arity function equivalent to the original function
+
+  // Replace known calls to the variadic with calls to the va_list equivalent
+  for (User *U : make_early_inc_range(VariadicWrapper->users())) {
+    if (CallBase *CB = dyn_cast<CallBase>(U)) {
+      Value *calledOperand = CB->getCalledOperand();
+      if (VariadicWrapper == calledOperand)
+        Changed |=
+            expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(),
+                       FixedArityReplacement);
+    }
+  }
+
+  // The original function will be erased.
+  // One of the two new functions will become a replacement for the original.
+  // When preserving the ABI, the other is an internal implementation detail.
+  // When rewriting the ABI, RAUW then the variadic one.
+  Function *const ExternallyAccessible =
+      rewriteABI() ? FixedArityReplacement : VariadicWrapper;
+  Function *const InternalOnly =
+      rewriteABI() ? VariadicWrapper : FixedArityReplacement;
+
+  // The external function is the replacement for the original
+  ExternallyAccessible->setLinkage(OriginalFunction->getLinkage());
+  ExternallyAccessible->setVisibility(OriginalFunction->getVisibility());
+  ExternallyAccessible->setComdat(OriginalFunction->getComdat());
+  ExternallyAccessible->takeName(OriginalFunction);
+
+  // Annotate the internal one as internal
+  InternalOnly->setVisibility(GlobalValue::DefaultVisibility);
+  InternalOnly->setLinkage(GlobalValue::InternalLinkage);
+
+  // The original is unused and obsolete
+  OriginalFunction->eraseFromParent();
+
+  InternalOnly->removeDeadConstantUsers();
+
+  if (rewriteABI()) {
+    // All known calls to the function have been removed by expandCall
+    // Resolve everything else by replaceAllUsesWith
+    VariadicWrapper->replaceAllUsesWith(FixedArityReplacement);
+    VariadicWrapper->eraseFromParent();
+  }
+
+  return Changed;
+}
+
+Function *
+ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,
+                                                  Function *OriginalFunction) {
+  auto &Ctx = M.getContext();
+  Function &F = *OriginalFunction;
+  FunctionType *FTy = F.getFunctionType();
+  Function *NF = Function::Create(FTy, F.getLinkage(), F.getAddressSpace());
+
+  NF->setName(F.getName() + ".varargs");
+  NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
+
+  F.getParent()->getFunctionList().insert(F.getIterator(), NF);
+
+  AttrBuilder ParamAttrs(Ctx);
+  AttributeList Attrs = NF->getAttributes();
+  Attrs = Attrs.addParamAttributes(Ctx, FTy->getNumParams(), ParamAttrs);
+  NF->setAttributes(Attrs);
+
+  OriginalFunction->replaceAllUsesWith(NF);
+  return NF;
+}
+
+Function *
+ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
+                                             Function *OriginalFunction) {
+  Function &F = *OriginalFunction;
+  // The purpose here is split the variadic function F into two functions
+  // One is a variadic function that bundles the passed argument into a va_list
+  // and passes it to the second function. The second function does whatever
+  // the original F does, except that it takes a va_list instead of the ...
+
+  assert(expansionApplicableToFunction(M, &F));
+
+  auto &Ctx = M.getContext();
+
+  // Returned value isDeclaration() is equal to F.isDeclaration()
+  // but that property is not invariant throughout this function
+  const bool FunctionIsDefinition = !F.isDeclaration();
+
+  FunctionType *FTy = F.getFunctionType();
+  SmallVector<Type *> ArgTypes(FTy->param_begin(), FTy->param_end());
+  ArgTypes.push_back(ABI->vaListParameterType(M));
+
+  FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);
+  Function *NF = Function::Create(NFTy, F.getLinkage(), F.getAddressSpace());
+
+  // Note - same attribute handling as DeadArgumentElimination
+  NF->copyAttributesFrom(&F);
+  NF->setComdat(F.getComdat());
+  F.getParent()->getFunctionList().insert(F.getIterator(), NF);
+  NF->setName(F.getName() + ".valist");
+  NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
+
+  AttrBuilder ParamAttrs(Ctx);
+
+  AttributeList Attrs = NF->getAttributes();
+  Attrs = Attrs.addParamAttributes(Ctx, NFTy->getNumParams() - 1, ParamAttrs);
+  NF->setAttributes(Attrs);
+
+  // Splice the implementation into the new function with minimal changes
+  if (FunctionIsDefinition) {
+    NF->splice(NF->begin(), &F);
+
+    auto NewArg = NF->arg_begin();
+    for (Argument &Arg : F.args()) {
+      Arg.replaceAllUsesWith(NewArg);
+      NewArg->setName(Arg.getName()); // takeName without killing the old one
+      ++NewArg;
+    }
+    NewArg->setName("varargs");
+  }
+
+  SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
+  F.getAllMetadata(MDs);
+  for (auto [KindID, Node] : MDs)
+    NF->addMetadata(KindID, *Node);
+  F.clearMetadata();
+
+  return NF;
+}
+
+Function *
+ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
+                                       Function *VariadicWrapper,
+                                       Function *FixedArityReplacement) {
+  auto &Ctx = Builder.getContext();
+  const DataLayout &DL = M.getDataLayout();
+  assert(VariadicWrapper->isDeclaration());
+  Function &F = *VariadicWrapper;
+
+  assert(F.isDeclaration());
+  Type *VaListTy = ABI->vaListType(Ctx);
+
+  auto *BB = BasicBlock::Create(Ctx, "entry", &F);
+  Builder.SetInsertPoint(BB);
+
+  AllocaInst *VaListInstance =
+      Builder.CreateAlloca(VaListTy, nullptr, "va_start");
+
+  Builder.CreateLifetimeStart(VaListInstance,
+                              sizeOfAlloca(Ctx, DL, VaListInstance));
+
+  Builder.CreateIntrinsic(Intrinsic::vastart, {DL.getAllocaPtrType(Ctx)},
+                          {VaListInstance});
+
+  SmallVector<Value *> Args;
+  for (Argument &A : F.args())
+    Args.push_back(&A);
+
+  Type *ParameterType = ABI->vaListParameterType(M);
+  if (ABI->vaListPassedInSSARegister())
+    Args.push_back(Builder.CreateLoad(ParameterType, VaListInstance));
+  else
+    Args.push_back(Builder.CreateAddrSpaceCast(VaListInstance, ParameterType));
+
+  CallInst *Result = Builder.CreateCall(FixedArityReplacement, Args);
+
+  Builder.CreateIntrinsic(Intrinsic::vaend, {DL.getAllocaPtrType(Ctx)},
+                          {VaListInstance});
+  Builder.CreateLifetimeEnd(VaListInstance,
+                            sizeOfAlloca(Ctx, DL, VaListInstance));
+
+  if (Result->getType()->isVoidTy())
+    Builder.CreateRetVoid();
+  else
+    Builder.CreateRet(Result);
+
+  return VariadicWrapper;
+}
+
+bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
+                                 FunctionType *VarargFunctionType,
+                                 Function *NF) {
+  bool Changed = false;
+  const DataLayout &DL = M.getDataLayout();
+
+  if (!expansionApplicableToFunctionCall(CB)) {
+    if (rewriteABI())
+      report_fatal_error("Cannot lower callbase instruction");
+    return Changed;
+  }
+
+  // This is tricky. The call instruction's function type might not match
+  // the type of the caller. When optimising, can leave it unchanged.
+  // Webassembly detects that inconsistency and repairs it.
+  FunctionType *FuncType = CB->getFunctionType();
+  if (FuncType != VarargFunctionType) {
+    if (!rewriteABI())
+      return Changed;
+    FuncType = VarargFunctionType;
+  }
+
+  auto &Ctx = CB->getContext();
+
+  Align MaxFieldAlign(1);
+
+  // The strategy is to allocate a call frame containing the variadic
+  // arguments laid out such that a target specific va_list can be initialized
+  // with it, such that target specific va_arg instructions will correctly
+  // iterate over it. This means getting the alignment right and sometimes
+  // embedding a pointer to the value instead of embedding the value itself.
+
+  Function *CBF = CB->getParent()->getParent();
+
+  ExpandedCallFrame Frame;
+
+  uint64_t CurrentOffset = 0;
+
+  for (unsigned I = FuncType->getNumParams(), E = CB->arg_size(); I < E; ++I) {
+    Value *ArgVal = CB->getArgOperand(I);
+    const bool IsByVal = CB->paramHasAttr(I, Attribute::ByVal);
+    const bool IsByRef = CB->paramHasAttr(I, Attribute::ByRef);
+
+    // The type of the value being passed, decoded from byval/byref metadata if
+    // required
+    Type *const UnderlyingType = IsByVal   ? CB->getParamByValType(I)
+                                 : IsByRef ? CB->getParamByRefType(I)
+                                           : ArgVal->getType();
+    const uint64_t UnderlyingSize =
+        DL.getTypeAllocSize(UnderlyingType).getFixedValue();
+
+    // The type to be written into the call frame
+    Type *FrameFieldType = UnderlyingType;
+
+    // The value to copy from when initialising the frame alloca
+    Value *SourceValue = ArgVal;
+
+    VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, UnderlyingType);
+
+    if (SlotInfo.Indirect) {
+      // The va_arg lowering loads through a pointer. Set up an alloca to aim
+      // that pointer at.
+      Builder.SetInsertPointPastAllocas(CBF);
+      Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
+      Value *CallerCopy =
+          Builder.CreateAlloca(UnderlyingType, nullptr, "IndirectAlloca");
+
+      Builder.SetInsertPoint(CB);
+      if (IsByVal)
+        Builder.CreateMemCpy(CallerCopy, {}, ArgVal, {}, UnderlyingSize);
+      else
+        Builder.CreateStore(ArgVal, CallerCopy);
+
+      // Indirection now handled, pass the alloca ptr by value
+      FrameFieldType = DL.getAllocaPtrType(Ctx);
+      SourceValue = CallerCopy;
+    }
+
+    // Alignment of the value within the frame
+    // This probably needs to be controllable as a function of type
+    Align DataAlign = SlotInfo.DataAlign;
+
+    MaxFieldAlign = std::max(MaxFieldAlign, DataAlign);
+
+    uint64_t DataAlignV = DataAlign.value();
+    if (uint64_t Rem = CurrentOffset % DataAlignV) {
+      // Inject explicit padding to deal with alignment requirements
+      uint64_t Padding = DataAlignV - Rem;
+      Frame.padding(Ctx, Padding);
+      CurrentOffset += Padding;
+    }
+
+    if (SlotInfo.Indirect) {
+      Frame.store(Ctx, FrameFieldType, SourceValue);
+    } else {
+      if (IsByVal)
+        Frame.memcpy(Ctx, FrameFieldType, SourceValue, UnderlyingSize);
+      else
+        Frame.store(Ctx, FrameFieldType, SourceValue);
+    }
+
+    CurrentOffset += DL.getTypeAllocSize(FrameFieldType).getFixedValue();
+  }
+
+  if (Frame.empty()) {
+    // Not passing any arguments, hopefully va_arg won't try to read any
+    // Creating a single byte frame containing nothing to point the va_list
+    // instance as that is less special-casey in the compiler and probably
+    // easier to interpret in a debugger.
+    Frame.padding(Ctx, 1);
+  }
+
+  StructType *VarargsTy = Frame.asStruct(Ctx, CBF->getName());
+
+  // The struct instance needs to be at least MaxFieldAlign for the alignment of
+  // the fields to be correct at runtime. Use the native stack alignment instead
+  // if that's greater as that tends to give better codegen.
+  // This is an awkward way to guess whether there is a known stack alignment
+  // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary
+  // number likely to be greater than the natural stack alignment.
+  // TODO: DL.getStackAlignment could return a MaybeAlign instead of assert
+  Align AllocaAlign = MaxFieldAlign;
+  if (DL.exceedsNaturalStackAlignment(Align(1024)))
+    AllocaAlign = std::max(AllocaAlign, DL.getStackAlignment());
+
+  // Put the alloca to hold the variadic args in the entry basic block.
+  Builder.SetInsertPointPastAllocas(CBF);
+
+  // SetCurrentDebugLocation when the builder SetInsertPoint method does not
+  Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
+
+  // The awkward construction here is to set the alignment on the instance
+  AllocaInst *Alloced = Builder.Insert(
+      new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign),
+      "vararg_buffer");
+  Changed = true;
+  assert(Alloced->getAllocatedType() == VarargsTy);
+
+  // Initialize the fields in the struct
+  Builder.SetInsertPoint(CB);
+  Builder.CreateLifetimeStart(Alloced, sizeOfAlloca(Ctx, DL, Alloced));
+  Frame.initializeStructAlloca(DL, Builder, Alloced);
+
+  const unsigned NumArgs = FuncType->getNumParams();
+  SmallVector<Value *> Args(CB->arg_begin(), CB->arg_begin() + NumArgs);
+
+  // Initialize a va_list pointing to that struct and pass it as the last
+  // argument
+  AllocaInst *VaList = nullptr;
+  {
+    if (!ABI->vaListPassedInSSARegister()) {
+      Type *VaListTy = ABI->vaListType(Ctx);
+      Builder.SetInsertPointPastAllocas(CBF);
+      Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
+      VaList = Builder.CreateAlloca(VaListTy, nullptr, "va_argument");
+      Builder.SetInsertPoint(CB);
+      Builder.CreateLifetimeStart(VaList, sizeOfAlloca(Ctx, DL, VaList));
+    }
+    Builder.SetInsertPoint(CB);
+    Args.push_back(ABI->initializeVaList(M, Ctx, Builder, VaList, Alloced));
+  }
+
+  // Attributes excluding any on the vararg arguments
+  AttributeList PAL = CB->getAttributes();
+  if (!PAL.isEmpty()) {
+    SmallVector<AttributeSet, 8> ArgAttrs;
+    for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)
+      ArgAttrs.push_back(PAL.getParamAttrs(ArgNo));
+    PAL =
+        AttributeList::get(Ctx, PAL.getFnAttrs(), PAL.getRetAttrs(), ArgAttrs);
+  }
+
+  SmallVector<OperandBundleDef, 1> OpBundles;
+  CB->getOperandBundlesAsDefs(OpBundles);
+
+  CallBase *NewCB = nullptr;
+
+  if (CallInst *CI = dyn_cast<CallInst>(CB)) {
+    Value *Dst = NF ? NF : CI->getCalledOperand();
+    FunctionType *NFTy = inlinableVariadicFunctionType(M, VarargFunctionType);
+
+    NewCB = CallInst::Create(NFTy, Dst, Args, OpBundles, "", CI);
+
+    CallInst::TailCallKind TCK = CI->getTailCallKind();
+    assert(TCK != CallInst::TCK_MustTail);
+
+    // Can't tail call a function that is being passed a pointer to an alloca
+    if (TCK == CallInst::TCK_Tail)
+      TCK = CallInst::TCK_None;
+    CI->setTailCallKind(TCK);
+
+  } else {
+    llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()");
+  }
+
+  if (VaList)
+    Builder.CreateLifetimeEnd(VaList, sizeOfAlloca(Ctx, DL, VaList));
+
+  Builder.CreateLifetimeEnd(Alloced, sizeOfAlloca(Ctx, DL, Alloced));
+
+  NewCB->setAttributes(PAL);
+  NewCB->takeName(CB);
+  NewCB->setCallingConv(CB->getCallingConv());
+  NewCB->setDebugLoc(DebugLoc());
+
+  // DeadArgElim and ArgPromotion copy exactly this metadata
+  NewCB->copyMetadata(*CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
+
+  CB->replaceAllUsesWith(NewCB);
+  CB->eraseFromParent();
+  return Changed;
+}
+
+bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
+                                            const DataLayout &DL,
+                                            VAStartInst *Inst) {
+  // Only removing va_start instructions that are not in variadic functions.
+  // Those would be rejected by the IR verifier before this pass.
+  // After splicing basic blocks from a variadic function into a fixed arity
+  // one the va_start that used to refer to the ... parameter still exist.
+  // There are also variadic functions that this pass did not change and
+  // va_start instances in the created single block wrapper functions.
+  // Replace exactly the instances in non-variadic functions as those are
+  // the ones to be fixed up to use the va_list passed as the final argument.
+
+  Function *ContainingFunction = Inst->getFunction();
+  if (ContainingFunction->isVarArg()) {
+    return false;
+  }
+
+  // The last argument is a vaListParameterType, either a va_list
+  // or a pointer to one depending on the target.
+  bool PassedByValue = ABI->vaListPassedInSSARegister();
+  Argument *PassedVaList =
+      ContainingFunction->getArg(ContainingFunction->arg_size() - 1);
+
+  // va_start takes a pointer to a va_list, e.g. one on the stack
+  Value *VaStartArg = Inst->getArgList();
+
+  Builder.SetInsertPoint(Inst);
+
+  if (PassedByValue) {
+    // The general thing to do is create an alloca, store the va_list argument
+    // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a
+    // store to the VaStartArg.
+    assert(ABI->vaCopyIsMemcpy());
+    Builder.CreateStore(PassedVaList, VaStartArg);
+  } else {
+
+    // Otherwise emit a vacopy to pick up target-specific handling if any
+    auto &Ctx = Builder.getContext();
+
+    Builder.CreateIntrinsic(Intrinsic::vacopy, {DL.getAllocaPtrType(Ctx)},
+                            {VaStartArg, PassedVaList});
+  }
+
+  Inst->eraseFromParent();
+  return true;
+}
+
+bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
+                                            VAEndInst *Inst) {
+  assert(ABI->vaEndIsNop());
+  Inst->eraseFromParent();
+  return true;
+}
+
+bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
+                                            const DataLayout &DL,
+                                            VACopyInst *Inst) {
+  assert(ABI->vaCopyIsMemcpy());
+  Builder.SetInsertPoint(Inst);
+
+  auto &Ctx = Builder.getContext();
+  Type *VaListTy = ABI->vaListType(Ctx);
+  uint64_t Size = DL.getTypeAllocSize(VaListTy).getFixedValue();
+
+  Builder.CreateMemCpy(Inst->getDest(), {}, Inst->getSrc(), {},
+                       Builder.getInt32(Size));
+
+  Inst->eraseFromParent();
+  return true;
+}
+
+struct Amdgpu final : public VariadicABIInfo {
+
+  bool enableForTarget() override { return true; }
+
+  bool vaListPassedInSSARegister() override { return true; }
+
+  Type *vaListType(LLVMContext &Ctx) override {
+    return PointerType::getUnqual(Ctx);
+  }
+
+  Type *vaListParameterType(Module &M) override {
+    return PointerType::getUnqual(M.getContext());
+  }
+
+  Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
+                          AllocaInst * /*va_list*/, Value *Buffer) override {
+    // Given Buffer, which is an AllocInst of vararg_buffer
+    // need to return something usable as parameter type
+    return Builder.CreateAddrSpaceCast(Buffer, vaListParameterType(M));
+  }
+
+  VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
+    return {Align(4), false};
+  }
+};
+
+struct Wasm final : public VariadicABIInfo {
+
+  bool enableForTarget() override {
+    // Currently wasm is only used for testing.
+    return commandLineOverride();
+  }
+
+  bool vaListPassedInSSARegister() override { return true; }
+
+  Type *vaListType(LLVMContext &Ctx) override {
+    return PointerType::getUnqual(Ctx);
+  }
+
+  Type *vaListParameterType(Module &M) override {
+    return PointerType::getUnqual(M.getContext());
+  }
+
+  Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
+                          AllocaInst * /*va_list*/, Value *Buffer) override {
+    return Buffer;
+  }
+
+  VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
+    LLVMContext &Ctx = Parameter->getContext();
+    const unsigned MinAlign = 4;
+    Align A = DL.getABITypeAlign(Parameter);
+    if (A < MinAlign)
+      A = Align(MinAlign);
+
+    if (auto s = dyn_cast<StructType>(Parameter)) {
+      if (s->getNumElements() > 1) {
+        return {DL.getABITypeAlign(PointerType::getUnqual(Ctx)), true};
+      }
+    }
+
+    return {A, false};
+  }
+};
+
+std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(const Triple &T) {
+  switch (T.getArch()) {
+  case Triple::r600:
+  case Triple::amdgcn: {
+    return std::make_unique<Amdgpu>();
+  }
+
+  case Triple::wasm32: {
+    return std::make_unique<Wasm>();
+  }
+
+  default:
+    return {};
+  }
+}
+
+} // namespace
+
+char ExpandVariadics::ID = 0;
+
+INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false,
+                false)
+
+ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) {
+  return new ExpandVariadics(M);
+}
+
+PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) {
+  return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none()
+                                              : PreservedAnalyses::all();
+}
+
+ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {}
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 03923b83cf34c5493baf3c45d2085f6a2c345a74..b58b906465e569bc1a1af2ef9171e065dd2f37a7 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -262,8 +262,70 @@ public:
     // TODO: Should this be a map (from Caller node) for more efficient lookup?
     std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
 
-    // The set of IDs for contexts including this node.
-    DenseSet<uint32_t> ContextIds;
+    // Get the list of edges from which we can compute allocation information
+    // such as the context ids and allocation type of this node.
+    const std::vector<std::shared_ptr<ContextEdge>> *
+    getEdgesWithAllocInfo() const {
+      // If node has any callees, compute from those, otherwise compute from
+      // callers (i.e. if this is the leaf allocation node).
+      if (!CalleeEdges.empty())
+        return &CalleeEdges;
+      if (!CallerEdges.empty()) {
+        // A node with caller edges but no callee edges must be the allocation
+        // node.
+        assert(IsAllocation);
+        return &CallerEdges;
+      }
+      return nullptr;
+    }
+
+    // Compute the context ids for this node from the union of its edge context
+    // ids.
+    DenseSet<uint32_t> getContextIds() const {
+      DenseSet<uint32_t> ContextIds;
+      auto *Edges = getEdgesWithAllocInfo();
+      if (!Edges)
+        return {};
+      unsigned Count = 0;
+      for (auto &Edge : *Edges)
+        Count += Edge->getContextIds().size();
+      ContextIds.reserve(Count);
+      for (auto &Edge : *Edges)
+        ContextIds.insert(Edge->getContextIds().begin(),
+                          Edge->getContextIds().end());
+      return ContextIds;
+    }
+
+    // Compute the allocation type for this node from the OR of its edge
+    // allocation types.
+    uint8_t computeAllocType() const {
+      auto *Edges = getEdgesWithAllocInfo();
+      if (!Edges)
+        return (uint8_t)AllocationType::None;
+      uint8_t BothTypes =
+          (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
+      uint8_t AllocType = (uint8_t)AllocationType::None;
+      for (auto &Edge : *Edges) {
+        AllocType |= Edge->AllocTypes;
+        // Bail early if alloc type reached both, no further refinement.
+        if (AllocType == BothTypes)
+          return AllocType;
+      }
+      return AllocType;
+    }
+
+    // The context ids set for this node is empty if its edge context ids are
+    // also all empty.
+    bool emptyContextIds() const {
+      auto *Edges = getEdgesWithAllocInfo();
+      if (!Edges)
+        return true;
+      for (auto &Edge : *Edges) {
+        if (!Edge->getContextIds().empty())
+          return false;
+      }
+      return true;
+    }
 
     // List of clones of this ContextNode, initially empty.
     std::vector<ContextNode *> Clones;
@@ -308,15 +370,11 @@ public:
     void printCall(raw_ostream &OS) const { Call.print(OS); }
 
     // True if this node was effectively removed from the graph, in which case
-    // its context id set, caller edges, and callee edges should all be empty.
+    // it should have an allocation type of None and empty context ids.
     bool isRemoved() const {
-      // Note that we can have non-empty context ids with empty caller and
-      // callee edges if the graph ends up with a single node.
-      if (ContextIds.empty())
-        assert(CalleeEdges.empty() && CallerEdges.empty() &&
-               "Context ids empty but at least one of callee and caller edges "
-               "were not!");
-      return ContextIds.empty();
+      assert((AllocTypes == (uint8_t)AllocationType::None) ==
+             emptyContextIds());
+      return AllocTypes == (uint8_t)AllocationType::None;
     }
 
     void dump() const;
@@ -344,7 +402,7 @@ public:
     ContextEdge(ContextNode *Callee, ContextNode *Caller, uint8_t AllocType,
                 DenseSet<uint32_t> ContextIds)
         : Callee(Callee), Caller(Caller), AllocTypes(AllocType),
-          ContextIds(ContextIds) {}
+          ContextIds(std::move(ContextIds)) {}
 
     DenseSet<uint32_t> &getContextIds() { return ContextIds; }
 
@@ -429,7 +487,8 @@ private:
   /// else to its callers. Also updates OrigNode's edges to remove any context
   /// ids moved to the newly created edge.
   void connectNewNode(ContextNode *NewNode, ContextNode *OrigNode,
-                      bool TowardsCallee);
+                      bool TowardsCallee,
+                      DenseSet<uint32_t> RemainingContextIds);
 
   /// Get the stack id corresponding to the given Id or Index (for IR this will
   /// return itself, for a summary index this will return the id recorded in the
@@ -958,7 +1017,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
 
   // Update alloc type and context ids for this MIB.
   AllocNode->AllocTypes |= (uint8_t)AllocType;
-  AllocNode->ContextIds.insert(LastContextId);
 
   // Now add or update nodes for each stack id in alloc's context.
   // Later when processing the stack ids on non-alloc callsites we will adjust
@@ -983,7 +1041,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
     auto Ins = StackIdSet.insert(StackId);
     if (!Ins.second)
       StackNode->Recursive = true;
-    StackNode->ContextIds.insert(LastContextId);
     StackNode->AllocTypes |= (uint8_t)AllocType;
     PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
     PrevNode = StackNode;
@@ -1034,7 +1091,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       // it resulted in any added ids to NextNode.
       if (!NewIdsToAdd.empty()) {
         Edge->getContextIds().insert(NewIdsToAdd.begin(), NewIdsToAdd.end());
-        NextNode->ContextIds.insert(NewIdsToAdd.begin(), NewIdsToAdd.end());
         UpdateCallers(NextNode, Visited, UpdateCallers);
       }
     }
@@ -1043,21 +1099,16 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   DenseSet<const ContextEdge *> Visited;
   for (auto &Entry : AllocationCallToContextNodeMap) {
     auto *Node = Entry.second;
-    // Update ids on the allocation nodes before calling the recursive
-    // update along caller edges, since this simplifies the logic during
-    // that traversal.
-    DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Node->ContextIds);
-    Node->ContextIds.insert(NewIdsToAdd.begin(), NewIdsToAdd.end());
     UpdateCallers(Node, Visited, UpdateCallers);
   }
 }
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
-    ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee) {
-  // Make a copy of the context ids, since this will be adjusted below as they
-  // are moved.
-  DenseSet<uint32_t> RemainingContextIds = NewNode->ContextIds;
+    ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee,
+    // This must be passed by value to make a copy since it will be adjusted
+    // as ids are moved.
+    DenseSet<uint32_t> RemainingContextIds) {
   auto &OrigEdges =
       TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
   // Increment iterator in loop so that we can remove edges as needed.
@@ -1076,15 +1127,15 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
       continue;
     }
     if (TowardsCallee) {
+      uint8_t NewAllocType = computeAllocType(NewEdgeContextIds);
       auto NewEdge = std::make_shared<ContextEdge>(
-          Edge->Callee, NewNode, computeAllocType(NewEdgeContextIds),
-          NewEdgeContextIds);
+          Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
       NewNode->CalleeEdges.push_back(NewEdge);
       NewEdge->Callee->CallerEdges.push_back(NewEdge);
     } else {
+      uint8_t NewAllocType = computeAllocType(NewEdgeContextIds);
       auto NewEdge = std::make_shared<ContextEdge>(
-          NewNode, Edge->Caller, computeAllocType(NewEdgeContextIds),
-          NewEdgeContextIds);
+          NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
       NewNode->CallerEdges.push_back(NewEdge);
       NewEdge->Caller->CalleeEdges.push_back(NewEdge);
     }
@@ -1103,6 +1154,51 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
   }
 }
 
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+static void checkEdge(
+    const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
+  // Confirm that alloc type is not None and that we have at least one context
+  // id.
+  assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
+  assert(!Edge->ContextIds.empty());
+}
+
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
+                      bool CheckEdges = true) {
+  if (Node->isRemoved())
+    return;
+#ifndef NDEBUG
+  // Compute node's context ids once for use in asserts.
+  auto NodeContextIds = Node->getContextIds();
+#endif
+  // Node's context ids should be the union of both its callee and caller edge
+  // context ids.
+  if (Node->CallerEdges.size()) {
+    DenseSet<uint32_t> CallerEdgeContextIds(
+        Node->CallerEdges.front()->ContextIds);
+    for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
+      if (CheckEdges)
+        checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
+      set_union(CallerEdgeContextIds, Edge->ContextIds);
+    }
+    // Node can have more context ids than callers if some contexts terminate at
+    // node and some are longer.
+    assert(NodeContextIds == CallerEdgeContextIds ||
+           set_is_subset(CallerEdgeContextIds, NodeContextIds));
+  }
+  if (Node->CalleeEdges.size()) {
+    DenseSet<uint32_t> CalleeEdgeContextIds(
+        Node->CalleeEdges.front()->ContextIds);
+    for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
+      if (CheckEdges)
+        checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
+      set_union(CalleeEdgeContextIds, Edge->getContextIds());
+    }
+    assert(NodeContextIds == CalleeEdgeContextIds);
+  }
+}
+
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     assignStackNodesPostOrder(ContextNode *Node,
@@ -1178,7 +1274,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     // duplicated context ids. We have to recompute as we might have overlap
     // overlap between the saved context ids for different last nodes, and
     // removed them already during the post order traversal.
-    set_intersect(SavedContextIds, FirstNode->ContextIds);
+    set_intersect(SavedContextIds, FirstNode->getContextIds());
     ContextNode *PrevNode = nullptr;
     for (auto Id : Ids) {
       ContextNode *CurNode = getNodeForStackId(Id);
@@ -1211,18 +1307,17 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     ContextNode *NewNode = NodeOwner.back().get();
     NodeToCallingFunc[NewNode] = Func;
     NonAllocationCallToContextNodeMap[Call] = NewNode;
-    NewNode->ContextIds = SavedContextIds;
-    NewNode->AllocTypes = computeAllocType(NewNode->ContextIds);
+    NewNode->AllocTypes = computeAllocType(SavedContextIds);
 
     // Connect to callees of innermost stack frame in inlined call chain.
     // This updates context ids for FirstNode's callee's to reflect those
     // moved to NewNode.
-    connectNewNode(NewNode, FirstNode, /*TowardsCallee=*/true);
+    connectNewNode(NewNode, FirstNode, /*TowardsCallee=*/true, SavedContextIds);
 
     // Connect to callers of outermost stack frame in inlined call chain.
     // This updates context ids for FirstNode's caller's to reflect those
     // moved to NewNode.
-    connectNewNode(NewNode, LastNode, /*TowardsCallee=*/false);
+    connectNewNode(NewNode, LastNode, /*TowardsCallee=*/false, SavedContextIds);
 
     // Now we need to remove context ids from edges/nodes between First and
     // Last Node.
@@ -1234,18 +1329,32 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
 
       // Remove the context ids moved to NewNode from CurNode, and the
       // edge from the prior node.
-      set_subtract(CurNode->ContextIds, NewNode->ContextIds);
       if (PrevNode) {
         auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
         assert(PrevEdge);
-        set_subtract(PrevEdge->getContextIds(), NewNode->ContextIds);
+        set_subtract(PrevEdge->getContextIds(), SavedContextIds);
         if (PrevEdge->getContextIds().empty()) {
           PrevNode->eraseCallerEdge(PrevEdge);
           CurNode->eraseCalleeEdge(PrevEdge);
         }
       }
+      // Since we update the edges from leaf to tail, only look at the callee
+      // edges. This isn't an alloc node, so if there are no callee edges, the
+      // alloc type is None.
+      CurNode->AllocTypes = CurNode->CalleeEdges.empty()
+                                ? (uint8_t)AllocationType::None
+                                : CurNode->computeAllocType();
       PrevNode = CurNode;
     }
+    if (VerifyNodes) {
+      checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /*CheckEdges=*/true);
+      for (auto Id : Ids) {
+        ContextNode *CurNode = getNodeForStackId(Id);
+        // We should only have kept stack ids that had nodes.
+        assert(CurNode);
+        checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /*CheckEdges=*/true);
+      }
+    }
   }
 }
 
@@ -1319,7 +1428,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
 
     // Initialize the context ids with the last node's. We will subsequently
     // refine the context ids by computing the intersection along all edges.
-    DenseSet<uint32_t> LastNodeContextIds = LastNode->ContextIds;
+    DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
     assert(!LastNodeContextIds.empty());
 
     for (unsigned I = 0; I < Calls.size(); I++) {
@@ -1442,6 +1551,8 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
   DenseSet<const ContextNode *> Visited;
   for (auto &Entry : AllocationCallToContextNodeMap)
     assignStackNodesPostOrder(Entry.second, Visited, StackIdToMatchingCalls);
+  if (VerifyCCG)
+    check();
 }
 
 uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
@@ -1786,8 +1897,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
     // First check if we have already synthesized a node for this tail call.
     if (TailCallToContextNodeMap.count(NewCall)) {
       NewNode = TailCallToContextNodeMap[NewCall];
-      NewNode->ContextIds.insert(Edge->ContextIds.begin(),
-                                 Edge->ContextIds.end());
       NewNode->AllocTypes |= Edge->AllocTypes;
     } else {
       FuncToCallsWithMetadata[Func].push_back({NewCall});
@@ -1797,7 +1906,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
       NewNode = NodeOwner.back().get();
       NodeToCallingFunc[NewNode] = Func;
       TailCallToContextNodeMap[NewCall] = NewNode;
-      NewNode->ContextIds = Edge->ContextIds;
       NewNode->AllocTypes = Edge->AllocTypes;
     }
 
@@ -2091,6 +2199,8 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
   OS << "\n";
   OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
   OS << "\tContextIds:";
+  // Make a copy of the computed context ids that we can sort for stability.
+  auto ContextIds = getContextIds();
   std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
   std::sort(SortedIds.begin(), SortedIds.end());
   for (auto Id : SortedIds)
@@ -2150,53 +2260,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
   }
 }
 
-template <typename DerivedCCG, typename FuncTy, typename CallTy>
-static void checkEdge(
-    const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
-  // Confirm that alloc type is not None and that we have at least one context
-  // id.
-  assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
-  assert(!Edge->ContextIds.empty());
-}
-
-template <typename DerivedCCG, typename FuncTy, typename CallTy>
-static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
-                      bool CheckEdges = true) {
-  if (Node->isRemoved())
-    return;
-  // Node's context ids should be the union of both its callee and caller edge
-  // context ids.
-  if (Node->CallerEdges.size()) {
-    auto EI = Node->CallerEdges.begin();
-    auto &FirstEdge = *EI;
-    EI++;
-    DenseSet<uint32_t> CallerEdgeContextIds(FirstEdge->ContextIds);
-    for (; EI != Node->CallerEdges.end(); EI++) {
-      const auto &Edge = *EI;
-      if (CheckEdges)
-        checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
-      set_union(CallerEdgeContextIds, Edge->ContextIds);
-    }
-    // Node can have more context ids than callers if some contexts terminate at
-    // node and some are longer.
-    assert(Node->ContextIds == CallerEdgeContextIds ||
-           set_is_subset(CallerEdgeContextIds, Node->ContextIds));
-  }
-  if (Node->CalleeEdges.size()) {
-    auto EI = Node->CalleeEdges.begin();
-    auto &FirstEdge = *EI;
-    EI++;
-    DenseSet<uint32_t> CalleeEdgeContextIds(FirstEdge->ContextIds);
-    for (; EI != Node->CalleeEdges.end(); EI++) {
-      const auto &Edge = *EI;
-      if (CheckEdges)
-        checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
-      set_union(CalleeEdgeContextIds, Edge->ContextIds);
-    }
-    assert(Node->ContextIds == CalleeEdgeContextIds);
-  }
-}
-
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
   using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
@@ -2284,7 +2347,7 @@ struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
 
   static std::string getNodeAttributes(NodeRef Node, GraphType) {
     std::string AttributeString = (Twine("tooltip=\"") + getNodeId(Node) + " " +
-                                   getContextIds(Node->ContextIds) + "\"")
+                                   getContextIds(Node->getContextIds()) + "\"")
                                       .str();
     AttributeString +=
         (Twine(",fillcolor=\"") + getColor(Node->AllocTypes) + "\"").str();
@@ -2443,16 +2506,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     set_subtract(Edge->ContextIds, ContextIdsToMove);
     Edge->AllocTypes = computeAllocType(Edge->ContextIds);
   }
-  // Now perform some updates that are common to all cases: the NewCallee gets
-  // the moved ids added, and we need to remove those ids from OldCallee and
-  // update its alloc type (NewCallee alloc type updates handled above).
-  NewCallee->ContextIds.insert(ContextIdsToMove.begin(),
-                               ContextIdsToMove.end());
-  set_subtract(OldCallee->ContextIds, ContextIdsToMove);
-  OldCallee->AllocTypes = computeAllocType(OldCallee->ContextIds);
-  // OldCallee alloc type should be None iff its context id set is now empty.
-  assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
-         OldCallee->ContextIds.empty());
   // Now walk the old callee node's callee edges and move Edge's context ids
   // over to the corresponding edge into the clone (which is created here if
   // this is a newly created clone).
@@ -2484,6 +2537,12 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     NewCallee->CalleeEdges.push_back(NewEdge);
     NewEdge->Callee->CallerEdges.push_back(NewEdge);
   }
+  // Recompute the node alloc type now that its callee edges have been
+  // updated (since we will compute from those edges).
+  OldCallee->AllocTypes = OldCallee->computeAllocType();
+  // OldCallee alloc type should be None iff its context id set is now empty.
+  assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
+         OldCallee->emptyContextIds());
   if (VerifyCCG) {
     checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /*CheckEdges=*/false);
     checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /*CheckEdges=*/false);
@@ -2528,7 +2587,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
   DenseSet<const ContextNode *> Visited;
   for (auto &Entry : AllocationCallToContextNodeMap) {
     Visited.clear();
-    identifyClones(Entry.second, Visited, Entry.second->ContextIds);
+    identifyClones(Entry.second, Visited, Entry.second->getContextIds());
   }
   Visited.clear();
   for (auto &Entry : AllocationCallToContextNodeMap)
@@ -2714,7 +2773,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
   }
 
   // We should still have some context ids on the original Node.
-  assert(!Node->ContextIds.empty());
+  assert(!Node->emptyContextIds());
 
   // Sanity check that no alloc types on node or edges are None.
   assert(Node->AllocTypes != (uint8_t)AllocationType::None);
@@ -2918,7 +2977,7 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
       // find additional cloning is required.
       std::deque<ContextNode *> ClonesWorklist;
       // Ignore original Node if we moved all of its contexts to clones.
-      if (!Node->ContextIds.empty())
+      if (!Node->emptyContextIds())
         ClonesWorklist.push_back(Node);
       ClonesWorklist.insert(ClonesWorklist.end(), Node->Clones.begin(),
                             Node->Clones.end());
@@ -3258,7 +3317,7 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
 
     // Skip if either no call to update, or if we ended up with no context ids
     // (we moved all edges onto other clones).
-    if (!Node->hasCall() || Node->ContextIds.empty())
+    if (!Node->hasCall() || Node->emptyContextIds())
       return;
 
     if (Node->IsAllocation) {
diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
index 9a191b0e38bea15f34a2fe09e5ac8fc792428b09..3daa05a76d364321e490be5602c761a6322e6cd2 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
@@ -431,8 +431,8 @@ void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
       // and type of a callsite probe. This gets rid of the dependency on
       // plumbing a customized metadata through the codegen pipeline.
       uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
-          Index, Type, 0,
-          PseudoProbeDwarfDiscriminator::FullDistributionFactor);
+          Index, Type, 0, PseudoProbeDwarfDiscriminator::FullDistributionFactor,
+          DIL->getBaseDiscriminator());
       DIL = DIL->cloneWithDiscriminator(V);
       Call->setDebugLoc(DIL);
     }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 8205b49dfbe2f352483919ac1e86c4f2474bb170..0a73c58c074093181e109305fbd781119f8efd22 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -905,8 +905,14 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
 
   // (X | Op01C) + Op1C --> X + (Op01C + Op1C) iff the `or` is actually an `add`
   Constant *Op01C;
-  if (match(Op0, m_DisjointOr(m_Value(X), m_ImmConstant(Op01C))))
-    return BinaryOperator::CreateAdd(X, ConstantExpr::getAdd(Op01C, Op1C));
+  if (match(Op0, m_DisjointOr(m_Value(X), m_ImmConstant(Op01C)))) {
+    BinaryOperator *NewAdd =
+        BinaryOperator::CreateAdd(X, ConstantExpr::getAdd(Op01C, Op1C));
+    NewAdd->setHasNoSignedWrap(Add.hasNoSignedWrap() &&
+                               willNotOverflowSignedAdd(Op01C, Op1C, Add));
+    NewAdd->setHasNoUnsignedWrap(Add.hasNoUnsignedWrap());
+    return NewAdd;
+  }
 
   // (X | C2) + C --> (X | C2) ^ C2 iff (C2 == -C)
   const APInt *C2;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 89193f8ff94b6ef119e685338a9e2f3d056c3428..3fce4a3c20b335c724333aab6e5ee634bfcd17a8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2441,9 +2441,10 @@ Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp,
       Type *TruncTy = ShType->getWithNewBitWidth(TypeBits - Amt);
       Constant *NewC =
           ConstantInt::get(TruncTy, RHSC.ashr(*ShiftAmt).trunc(TypeBits - Amt));
-      return new ICmpInst(
-          CmpPred, Builder.CreateTrunc(X, TruncTy, "", Shl->hasNoSignedWrap()),
-          NewC);
+      return new ICmpInst(CmpPred,
+                          Builder.CreateTrunc(X, TruncTy, "", /*IsNUW=*/false,
+                                              Shl->hasNoSignedWrap()),
+                          NewC);
     }
   }
 
@@ -3502,15 +3503,13 @@ Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
     break;
   }
   case Instruction::Xor:
-    if (BO->hasOneUse()) {
-      if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
-        // For the xor case, we can xor two constants together, eliminating
-        // the explicit xor.
-        return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
-      } else if (C.isZero()) {
-        // Replace ((xor A, B) != 0) with (A != B)
-        return new ICmpInst(Pred, BOp0, BOp1);
-      }
+    if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
+      // For the xor case, we can xor two constants together, eliminating
+      // the explicit xor.
+      return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
+    } else if (C.isZero()) {
+      // Replace ((xor A, B) != 0) with (A != B)
+      return new ICmpInst(Pred, BOp0, BOp1);
     }
     break;
   case Instruction::Or: {
@@ -4745,6 +4744,29 @@ static Instruction *foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q,
                           Constant::getNullValue(Op1->getType()));
   }
 
+  if (!ICmpInst::isSigned(Pred))
+    return nullptr;
+
+  KnownBits KnownY = IC.computeKnownBits(A, /*Depth=*/0, &I);
+  // (X & NegY) spred X --> (X & NegY) upred X
+  if (KnownY.isNegative())
+    return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1);
+
+  if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGT)
+    return nullptr;
+
+  if (KnownY.isNonNegative())
+    // (X & PosY) s<= X --> X s>= 0
+    // (X & PosY) s> X --> X s< 0
+    return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
+                        Constant::getNullValue(Op1->getType()));
+
+  if (isKnownNegative(Op1, IC.getSimplifyQuery().getWithInstruction(&I)))
+    // (NegX & Y) s<= NegX --> Y s< 0
+    // (NegX & Y) s> NegX --> Y s>= 0
+    return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), A,
+                        Constant::getNullValue(A->getType()));
+
   return nullptr;
 }
 
@@ -4772,7 +4794,7 @@ static Instruction *foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q,
   if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
     // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
     if (Value *NotOp1 =
-            IC.getFreelyInverted(Op1, Op1->hasOneUse(), &IC.Builder))
+            IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
       return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1),
                           Constant::getNullValue(Op1->getType()));
     // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X  is freely invertible.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 313beb7b6407a1b032aa0bb14eba1747077732e4..b04e0b300f95acd8f62ccb4ba63ab9cf4ac4e03a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1288,22 +1288,36 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
     Swapped = true;
   }
 
-  // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand.
-  // Make sure Y cannot be undef though, as we might pick different values for
-  // undef in the icmp and in f(Y). Additionally, take care to avoid replacing
-  // X == Y ? X : Z with X == Y ? Y : Z, as that would lead to an infinite
-  // replacement cycle.
   Value *CmpLHS = Cmp.getOperand(0), *CmpRHS = Cmp.getOperand(1);
-  if (TrueVal != CmpLHS &&
-      isGuaranteedNotToBeUndefOrPoison(CmpRHS, SQ.AC, &Sel, &DT)) {
-    if (Value *V = simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, SQ,
-                                          /* AllowRefinement */ true))
-      // Require either the replacement or the simplification result to be a
-      // constant to avoid infinite loops.
-      // FIXME: Make this check more precise.
-      if (isa<Constant>(CmpRHS) || isa<Constant>(V))
+  auto ReplaceOldOpWithNewOp = [&](Value *OldOp,
+                                   Value *NewOp) -> Instruction * {
+    // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand.
+    // Take care to avoid replacing X == Y ? X : Z with X == Y ? Y : Z, as that
+    // would lead to an infinite replacement cycle.
+    // If we will be able to evaluate f(Y) to a constant, we can allow undef,
+    // otherwise Y cannot be undef as we might pick different values for undef
+    // in the icmp and in f(Y).
+    if (TrueVal == OldOp)
+      return nullptr;
+
+    if (Value *V = simplifyWithOpReplaced(TrueVal, OldOp, NewOp, SQ,
+                                          /* AllowRefinement=*/true)) {
+      // Need some guarantees about the new simplified op to ensure we don't inf
+      // loop.
+      // If we simplify to a constant, replace if we aren't creating new undef.
+      if (match(V, m_ImmConstant()) &&
+          isGuaranteedNotToBeUndef(V, SQ.AC, &Sel, &DT))
         return replaceOperand(Sel, Swapped ? 2 : 1, V);
 
+      // If NewOp is a constant and OldOp is not replace iff NewOp doesn't
+      // contain and undef elements.
+      if (match(NewOp, m_ImmConstant())) {
+        if (isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT))
+          return replaceOperand(Sel, Swapped ? 2 : 1, V);
+        return nullptr;
+      }
+    }
+
     // Even if TrueVal does not simplify, we can directly replace a use of
     // CmpLHS with CmpRHS, as long as the instruction is not used anywhere
     // else and is safe to speculatively execute (we may end up executing it
@@ -1311,17 +1325,18 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
     // undefined behavior). Only do this if CmpRHS is a constant, as
     // profitability is not clear for other cases.
     // FIXME: Support vectors.
-    if (match(CmpRHS, m_ImmConstant()) && !match(CmpLHS, m_ImmConstant()) &&
-        !Cmp.getType()->isVectorTy())
-      if (replaceInInstruction(TrueVal, CmpLHS, CmpRHS))
+    if (OldOp == CmpLHS && match(NewOp, m_ImmConstant()) &&
+        !match(OldOp, m_ImmConstant()) && !Cmp.getType()->isVectorTy() &&
+        isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT))
+      if (replaceInInstruction(TrueVal, OldOp, NewOp))
         return &Sel;
-  }
-  if (TrueVal != CmpRHS &&
-      isGuaranteedNotToBeUndefOrPoison(CmpLHS, SQ.AC, &Sel, &DT))
-    if (Value *V = simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, SQ,
-                                          /* AllowRefinement */ true))
-      if (isa<Constant>(CmpLHS) || isa<Constant>(V))
-        return replaceOperand(Sel, Swapped ? 2 : 1, V);
+    return nullptr;
+  };
+
+  if (Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS))
+    return R;
+  if (Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS))
+    return R;
 
   auto *FalseInst = dyn_cast<Instruction>(FalseVal);
   if (!FalseInst)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 9ff817da79368a60f28eb31c6d328f1971b69970..4a014ab6e044e5840991927772ddf56af60365fc 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -767,11 +767,20 @@ Instruction *InstCombinerImpl::FoldShiftByConstant(Value *Op0, Constant *C1,
   // (C2 >> X) >> C1 --> (C2 >> C1) >> X
   Constant *C2;
   Value *X;
-  if (match(Op0, m_BinOp(I.getOpcode(), m_ImmConstant(C2), m_Value(X))))
-    return BinaryOperator::Create(
+  bool IsLeftShift = I.getOpcode() == Instruction::Shl;
+  if (match(Op0, m_BinOp(I.getOpcode(), m_ImmConstant(C2), m_Value(X)))) {
+    Instruction *R = BinaryOperator::Create(
         I.getOpcode(), Builder.CreateBinOp(I.getOpcode(), C2, C1), X);
+    BinaryOperator *BO0 = cast<BinaryOperator>(Op0);
+    if (IsLeftShift) {
+      R->setHasNoUnsignedWrap(I.hasNoUnsignedWrap() &&
+                              BO0->hasNoUnsignedWrap());
+      R->setHasNoSignedWrap(I.hasNoSignedWrap() && BO0->hasNoSignedWrap());
+    } else
+      R->setIsExact(I.isExact() && BO0->isExact());
+    return R;
+  }
 
-  bool IsLeftShift = I.getOpcode() == Instruction::Shl;
   Type *Ty = I.getType();
   unsigned TypeBits = Ty->getScalarSizeInBits();
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 6739b8745d74e4e2e058b67c5057171fd45f00ae..b3eebee0d0d8c1a282d5befd4da9d32f9899c511 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -209,8 +209,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.Zero, LHSKnown,
                              Depth + 1))
       return I;
-    assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?");
-    assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?");
 
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
                                          Depth, SQ.getWithInstruction(CxtI));
@@ -242,8 +240,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
       I->dropPoisonGeneratingFlags();
       return I;
     }
-    assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?");
-    assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?");
 
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
                                          Depth, SQ.getWithInstruction(CxtI));
@@ -291,9 +287,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
       return Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, Xor);
     }
 
-    assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?");
-    assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?");
-
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
                                          Depth, SQ.getWithInstruction(CxtI));
 
@@ -375,8 +368,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     if (SimplifyDemandedBits(I, 2, DemandedMask, RHSKnown, Depth + 1) ||
         SimplifyDemandedBits(I, 1, DemandedMask, LHSKnown, Depth + 1))
       return I;
-    assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?");
-    assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?");
 
     // If the operands are constants, see if we can simplify them.
     // This is similar to ShrinkDemandedConstant, but for a select we want to
@@ -455,7 +446,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
       InputKnown.makeNonNegative();
     Known = InputKnown.zextOrTrunc(BitWidth);
 
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     break;
   }
   case Instruction::SExt: {
@@ -481,12 +471,11 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
       CastInst *NewCast = new ZExtInst(I->getOperand(0), VTy);
       NewCast->takeName(I);
       return InsertNewInstWith(NewCast, I->getIterator());
-     }
+    }
 
     // If the sign bit of the input is known set or clear, then we know the
     // top bits of the result.
     Known = InputKnown.sext(BitWidth);
-    assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     break;
   }
   case Instruction::Add: {
@@ -696,7 +685,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
 
       if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Depth + 1))
         return I;
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
 
       Known = KnownBits::shl(Known,
                              KnownBits::makeConstant(APInt(BitWidth, ShiftAmt)),
@@ -772,7 +760,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         I->dropPoisonGeneratingFlags();
         return I;
       }
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       Known.Zero.lshrInPlace(ShiftAmt);
       Known.One.lshrInPlace(ShiftAmt);
       if (ShiftAmt)
@@ -819,7 +806,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         return I;
       }
 
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
       // Compute the new bits that are at the top now plus sign bits.
       APInt HighBits(APInt::getHighBitsSet(
           BitWidth, std::min(SignBits + ShiftAmt - 1, BitWidth)));
@@ -900,7 +886,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         if (LHSKnown.isNegative() && LowBits.intersects(LHSKnown.One))
           Known.One |= ~LowBits;
 
-        assert(!Known.hasConflict() && "Bits known to be one AND zero?");
         break;
       }
     }
@@ -978,8 +963,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
 
         // TODO: Should be 1-extend
         RHSKnown = RHSKnown.anyextOrTrunc(BitWidth);
-        assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?");
-        assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?");
 
         Known = LHSKnown & RHSKnown;
         KnownBitsComputed = true;
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 2aa21759d56e02879c0d28836ef38ddcf608f906..a0e63bf12400e34f3fb9a8139209e823305bfb51 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -337,13 +337,17 @@ private:
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore, DomTreeUpdater &DTU,
                                  LoopInfo *LI);
-  bool ignoreMemIntrinsic(MemIntrinsic *MI);
+  bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI);
   void instrumentMemIntrinsic(MemIntrinsic *MI);
   bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
                            LoopInfo *LI);
-  bool ignoreAccess(Instruction *Inst, Value *Ptr);
+  bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr);
+  bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst,
+                    Value *Ptr);
+
   void getInterestingMemoryOperands(
-      Instruction *I, const TargetLibraryInfo &TLI,
+      OptimizationRemarkEmitter &ORE, Instruction *I,
+      const TargetLibraryInfo &TLI,
       SmallVectorImpl<InterestingMemoryOperand> &Interesting);
 
   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
@@ -765,7 +769,8 @@ Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
   return IRB.CreateLoad(PtrTy, GlobalDynamicAddress);
 }
 
-bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
+bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst,
+                                                   Value *Ptr) {
   // Do not instrument accesses from different address spaces; we cannot deal
   // with them.
   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
@@ -795,8 +800,23 @@ bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
   return false;
 }
 
+bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE,
+                                      Instruction *Inst, Value *Ptr) {
+  bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr);
+  if (Ignored) {
+    ORE.emit(
+        [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); });
+  } else {
+    ORE.emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst);
+    });
+  }
+  return Ignored;
+}
+
 void HWAddressSanitizer::getInterestingMemoryOperands(
-    Instruction *I, const TargetLibraryInfo &TLI,
+    OptimizationRemarkEmitter &ORE, Instruction *I,
+    const TargetLibraryInfo &TLI,
     SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
   // Skip memory accesses inserted by another instrumentation.
   if (I->hasMetadata(LLVMContext::MD_nosanitize))
@@ -807,22 +827,22 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
     return;
 
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-    if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand()))
+    if (!ClInstrumentReads || ignoreAccess(ORE, I, LI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
                              LI->getType(), LI->getAlign());
   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
-    if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand()))
+    if (!ClInstrumentWrites || ignoreAccess(ORE, I, SI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
                              SI->getValueOperand()->getType(), SI->getAlign());
   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
-    if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
+    if (!ClInstrumentAtomics || ignoreAccess(ORE, I, RMW->getPointerOperand()))
       return;
     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
                              RMW->getValOperand()->getType(), std::nullopt);
   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
-    if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
+    if (!ClInstrumentAtomics || ignoreAccess(ORE, I, XCHG->getPointerOperand()))
       return;
     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
                              XCHG->getCompareOperand()->getType(),
@@ -830,7 +850,7 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
   } else if (auto *CI = dyn_cast<CallInst>(I)) {
     for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
-          ignoreAccess(I, CI->getArgOperand(ArgNo)))
+          ignoreAccess(ORE, I, CI->getArgOperand(ArgNo)))
         continue;
       Type *Ty = CI->getParamByValType(ArgNo);
       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
@@ -1035,13 +1055,14 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
         ->setSuccessor(0, TCI.TagMismatchTerm->getParent());
 }
 
-bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) {
+bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE,
+                                            MemIntrinsic *MI) {
   if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
-    return (!ClInstrumentWrites || ignoreAccess(MTI, MTI->getDest())) &&
-           (!ClInstrumentReads || ignoreAccess(MTI, MTI->getSource()));
+    return (!ClInstrumentWrites || ignoreAccess(ORE, MTI, MTI->getDest())) &&
+           (!ClInstrumentReads || ignoreAccess(ORE, MTI, MTI->getSource()));
   }
   if (isa<MemSetInst>(MI))
-    return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest());
+    return !ClInstrumentWrites || ignoreAccess(ORE, MI, MI->getDest());
   return false;
 }
 
@@ -1541,6 +1562,9 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
 
   NumTotalFuncs++;
 
+  OptimizationRemarkEmitter &ORE =
+      FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+
   if (selectiveInstrumentationShouldSkip(F, FAM))
     return;
 
@@ -1562,10 +1586,10 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
     if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
       LandingPadVec.push_back(&Inst);
 
-    getInterestingMemoryOperands(&Inst, TLI, OperandsToInstrument);
+    getInterestingMemoryOperands(ORE, &Inst, TLI, OperandsToInstrument);
 
     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
-      if (!ignoreMemIntrinsic(MI))
+      if (!ignoreMemIntrinsic(ORE, MI))
         IntrinToInstrument.push_back(MI);
   }
 
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index c0a3bf8464d2d393463576962e3492af4f4aa904..aac57231ba2ed62c76c2e0923fed576c29d5412f 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -149,11 +149,33 @@ static cl::opt<bool> ClMemProfMatchHotColdNew(
         "Match allocation profiles onto existing hot/cold operator new calls"),
     cl::Hidden, cl::init(false));
 
+static cl::opt<bool>
+    ClPrintMemProfMatchInfo("memprof-print-match-info",
+                            cl::desc("Print matching stats for each allocation "
+                                     "context in this module's profiles"),
+                            cl::Hidden, cl::init(false));
+
+// Instrumentation statistics
 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
+
+// Matching statistics
 STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");
+STATISTIC(NumOfMemProfMismatch,
+          "Number of functions having mismatched memory profile hash.");
+STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile.");
+STATISTIC(NumOfMemProfAllocContextProfiles,
+          "Number of alloc contexts in memory profile.");
+STATISTIC(NumOfMemProfCallSiteProfiles,
+          "Number of callsites in memory profile.");
+STATISTIC(NumOfMemProfMatchedAllocContexts,
+          "Number of matched memory profile alloc contexts.");
+STATISTIC(NumOfMemProfMatchedAllocs,
+          "Number of matched memory profile allocs.");
+STATISTIC(NumOfMemProfMatchedCallSites,
+          "Number of matched memory profile callsites.");
 
 namespace {
 
@@ -637,8 +659,22 @@ static uint64_t computeStackId(const memprof::Frame &Frame) {
   return computeStackId(Frame.Function, Frame.LineOffset, Frame.Column);
 }
 
-static void addCallStack(CallStackTrie &AllocTrie,
-                         const AllocationInfo *AllocInfo) {
+// Helper to generate a single hash id for a given callstack, used for emitting
+// matching statistics and useful for uniquing such statistics across modules.
+static uint64_t
+computeFullStackId(const std::vector<memprof::Frame> &CallStack) {
+  llvm::HashBuilder<llvm::TruncatedBLAKE3<8>, llvm::endianness::little>
+      HashBuilder;
+  for (auto &F : CallStack)
+    HashBuilder.add(F.Function, F.LineOffset, F.Column);
+  llvm::BLAKE3Result<8> Hash = HashBuilder.final();
+  uint64_t Id;
+  std::memcpy(&Id, Hash.data(), sizeof(Hash));
+  return Id;
+}
+
+static AllocationType addCallStack(CallStackTrie &AllocTrie,
+                                   const AllocationInfo *AllocInfo) {
   SmallVector<uint64_t> StackIds;
   for (const auto &StackFrame : AllocInfo->CallStack)
     StackIds.push_back(computeStackId(StackFrame));
@@ -646,6 +682,7 @@ static void addCallStack(CallStackTrie &AllocTrie,
                                 AllocInfo->Info.getAllocCount(),
                                 AllocInfo->Info.getTotalLifetime());
   AllocTrie.addCallStack(AllocType, StackIds);
+  return AllocType;
 }
 
 // Helper to compare the InlinedCallStack computed from an instruction's debug
@@ -701,9 +738,16 @@ static bool isNewWithHotColdVariant(Function *Callee,
   }
 }
 
-static void readMemprof(Module &M, Function &F,
-                        IndexedInstrProfReader *MemProfReader,
-                        const TargetLibraryInfo &TLI) {
+struct AllocMatchInfo {
+  uint64_t TotalSize = 0;
+  AllocationType AllocType = AllocationType::None;
+  bool Matched = false;
+};
+
+static void
+readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
+            const TargetLibraryInfo &TLI,
+            std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo) {
   auto &Ctx = M.getContext();
   // Previously we used getIRPGOFuncName() here. If F is local linkage,
   // getIRPGOFuncName() returns FuncName with prefix 'FileName;'. But
@@ -727,6 +771,7 @@ static void readMemprof(Module &M, Function &F,
         SkipWarning = !PGOWarnMissing;
         LLVM_DEBUG(dbgs() << "unknown function");
       } else if (Err == instrprof_error::hash_mismatch) {
+        NumOfMemProfMismatch++;
         SkipWarning =
             NoPGOWarnMismatch ||
             (NoPGOWarnMismatchComdatWeak &&
@@ -748,6 +793,8 @@ static void readMemprof(Module &M, Function &F,
     return;
   }
 
+  NumOfMemProfFunc++;
+
   // Detect if there are non-zero column numbers in the profile. If not,
   // treat all column numbers as 0 when matching (i.e. ignore any non-zero
   // columns in the IR). The profiled binary might have been built with
@@ -759,9 +806,10 @@ static void readMemprof(Module &M, Function &F,
   std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;
   // For the callsites we need to record the index of the associated frame in
   // the frame array (see comments below where the map entries are added).
-  std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *, unsigned>>>
+  std::map<uint64_t, std::set<std::pair<const std::vector<Frame> *, unsigned>>>
       LocHashToCallSites;
   for (auto &AI : MemProfRec->AllocSites) {
+    NumOfMemProfAllocContextProfiles++;
     // Associate the allocation info with the leaf frame. The later matching
     // code will match any inlined call sequences in the IR with a longer prefix
     // of call stack frames.
@@ -770,6 +818,7 @@ static void readMemprof(Module &M, Function &F,
     ProfileHasColumns |= AI.CallStack[0].Column;
   }
   for (auto &CS : MemProfRec->CallSites) {
+    NumOfMemProfCallSiteProfiles++;
     // Need to record all frames from leaf up to and including this function,
     // as any of these may or may not have been inlined at this point.
     unsigned Idx = 0;
@@ -815,7 +864,7 @@ static void readMemprof(Module &M, Function &F,
       // and another callsite).
       std::map<uint64_t, std::set<const AllocationInfo *>>::iterator
           AllocInfoIter;
-      std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *,
+      std::map<uint64_t, std::set<std::pair<const std::vector<Frame> *,
                                             unsigned>>>::iterator CallSitesIter;
       for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;
            DIL = DIL->getInlinedAt()) {
@@ -863,13 +912,23 @@ static void readMemprof(Module &M, Function &F,
           // If we found and thus matched all frames on the call, include
           // this MIB.
           if (stackFrameIncludesInlinedCallStack(AllocInfo->CallStack,
-                                                 InlinedCallStack))
-            addCallStack(AllocTrie, AllocInfo);
+                                                 InlinedCallStack)) {
+            NumOfMemProfMatchedAllocContexts++;
+            auto AllocType = addCallStack(AllocTrie, AllocInfo);
+            // Record information about the allocation if match info printing
+            // was requested.
+            if (ClPrintMemProfMatchInfo) {
+              auto FullStackId = computeFullStackId(AllocInfo->CallStack);
+              FullStackIdToAllocMatchInfo[FullStackId] = {
+                  AllocInfo->Info.getTotalSize(), AllocType, /*Matched=*/true};
+            }
+          }
         }
         // We might not have matched any to the full inlined call stack.
         // But if we did, create and attach metadata, or a function attribute if
         // all contexts have identical profiled behavior.
         if (!AllocTrie.empty()) {
+          NumOfMemProfMatchedAllocs++;
           // MemprofMDAttached will be false if a function attribute was
           // attached.
           bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata(CI);
@@ -897,6 +956,7 @@ static void readMemprof(Module &M, Function &F,
         // attach call stack metadata.
         if (stackFrameIncludesInlinedCallStack(
                 *CallStackIdx.first, InlinedCallStack, CallStackIdx.second)) {
+          NumOfMemProfMatchedCallSites++;
           addCallsiteMetadata(I, InlinedCallStack, Ctx);
           // Only need to find one with a matching call stack and add a single
           // callsite metadata.
@@ -942,12 +1002,25 @@ PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) {
 
   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 
+  // Map from the stack has of each allocation context in the function profiles
+  // to the total profiled size (bytes), allocation type, and whether we matched
+  // it to an allocation in the IR.
+  std::map<uint64_t, AllocMatchInfo> FullStackIdToAllocMatchInfo;
+
   for (auto &F : M) {
     if (F.isDeclaration())
       continue;
 
     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
-    readMemprof(M, F, MemProfReader.get(), TLI);
+    readMemprof(M, F, MemProfReader.get(), TLI, FullStackIdToAllocMatchInfo);
+  }
+
+  if (ClPrintMemProfMatchInfo) {
+    for (const auto &[Id, Info] : FullStackIdToAllocMatchInfo)
+      errs() << "MemProf " << getAllocTypeAttributeString(Info.AllocType)
+             << " context with id " << Id << " has total profiled size "
+             << Info.TotalSize << (Info.Matched ? " is" : " not")
+             << " matched\n";
   }
 
   return PreservedAnalyses::none();
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index fd0f69eca96e6c742dc7bd7772f96498ba64e17c..fa93f4bd63ce67d2a25d94671c81835c4f99bb24 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -402,8 +402,7 @@ bool MemOPSizeOpt::perform(MemOp MO) {
   // If all promoted, we don't need the MD.prof metadata.
   if (SavedRemainCount > 0 || Version != NumVals) {
     // Otherwise we need update with the un-promoted records back.
-    ArrayRef<InstrProfValueData> RemVDs(RemainingVDs);
-    annotateValueSite(*Func.getParent(), *MO.I, RemVDs, SavedRemainCount,
+    annotateValueSite(*Func.getParent(), *MO.I, RemainingVDs, SavedRemainCount,
                       IPVK_MemOPSize, NumVals);
   }
 
diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
index ba2546b8db0e22a1295a61997207b9180f837b5e..4371b821eae63f23c4c071b222a1b9a93600ebc2 100644
--- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
@@ -827,7 +827,8 @@ private:
         return false;
       }
 
-      if (Metrics.convergent) {
+      // FIXME: Allow jump threading with controlled convergence.
+      if (Metrics.Convergence != ConvergenceKind::None) {
         LLVM_DEBUG(dbgs() << "DFA Jump Threading: Not jump threading, contains "
                           << "convergent instructions.\n");
         ORE->emit([&]() {
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index 7b4c54370e48a7bc6794258aa8b9330b933a4fa4..f8e2f1f28088d8a420592e3f94f55023bba54879 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -327,8 +327,7 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
   UnrollCostEstimator OuterUCE(L, TTI, EphValues, UP.BEInsns);
 
   if (!InnerUCE.canUnroll() || !OuterUCE.canUnroll()) {
-    LLVM_DEBUG(dbgs() << "  Not unrolling loop which contains instructions"
-                      << " which cannot be duplicated or have invalid cost.\n");
+    LLVM_DEBUG(dbgs() << "  Loop not considered unrollable\n");
     return LoopUnrollResult::Unmodified;
   }
 
@@ -341,7 +340,10 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
     LLVM_DEBUG(dbgs() << "  Not unrolling loop with inlinable calls.\n");
     return LoopUnrollResult::Unmodified;
   }
-  if (InnerUCE.Convergent || OuterUCE.Convergent) {
+  // FIXME: The call to canUnroll() allows some controlled convergent
+  // operations, but we block them here for future changes.
+  if (InnerUCE.Convergence != ConvergenceKind::None ||
+      OuterUCE.Convergence != ConvergenceKind::None) {
     LLVM_DEBUG(
         dbgs() << "  Not unrolling loop with convergent instructions.\n");
     return LoopUnrollResult::Unmodified;
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 10fc9e9303e899ee9169ff0ab10c924d99145291..cbc35b6dd4292aa8c8d4418bae5180e6d040279e 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -684,11 +684,15 @@ UnrollCostEstimator::UnrollCostEstimator(
     const SmallPtrSetImpl<const Value *> &EphValues, unsigned BEInsns) {
   CodeMetrics Metrics;
   for (BasicBlock *BB : L->blocks())
-    Metrics.analyzeBasicBlock(BB, TTI, EphValues);
+    Metrics.analyzeBasicBlock(BB, TTI, EphValues, /* PrepareForLTO= */ false,
+                              L);
   NumInlineCandidates = Metrics.NumInlineCandidates;
   NotDuplicatable = Metrics.notDuplicatable;
-  Convergent = Metrics.convergent;
+  Convergence = Metrics.Convergence;
   LoopSize = Metrics.NumInsts;
+  ConvergenceAllowsRuntime =
+      Metrics.Convergence != ConvergenceKind::Uncontrolled &&
+      !getLoopConvergenceHeart(L);
 
   // Don't allow an estimate of size zero.  This would allows unrolling of loops
   // with huge iteration counts, which is a compile time problem even if it's
@@ -701,6 +705,25 @@ UnrollCostEstimator::UnrollCostEstimator(
     LoopSize = BEInsns + 1;
 }
 
+bool UnrollCostEstimator::canUnroll() const {
+  switch (Convergence) {
+  case ConvergenceKind::ExtendedLoop:
+    LLVM_DEBUG(dbgs() << "  Convergence prevents unrolling.\n");
+    return false;
+  default:
+    break;
+  }
+  if (!LoopSize.isValid()) {
+    LLVM_DEBUG(dbgs() << "  Invalid loop size prevents unrolling.\n");
+    return false;
+  }
+  if (NotDuplicatable) {
+    LLVM_DEBUG(dbgs() << "  Non-duplicatable blocks prevent unrolling.\n");
+    return false;
+  }
+  return true;
+}
+
 uint64_t UnrollCostEstimator::getUnrolledLoopSize(
     const TargetTransformInfo::UnrollingPreferences &UP,
     unsigned CountOverwrite) const {
@@ -1206,8 +1229,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
 
   UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
   if (!UCE.canUnroll()) {
-    LLVM_DEBUG(dbgs() << "  Not unrolling loop which contains instructions"
-                      << " which cannot be duplicated or have invalid cost.\n");
+    LLVM_DEBUG(dbgs() << "  Loop not considered unrollable.\n");
     return LoopUnrollResult::Unmodified;
   }
 
@@ -1254,15 +1276,9 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
   // is unsafe -- it adds a control-flow dependency to the convergent
   // operation.  Therefore restrict remainder loop (try unrolling without).
   //
-  // TODO: This is quite conservative.  In practice, convergent_op()
-  // is likely to be called unconditionally in the loop.  In this
-  // case, the program would be ill-formed (on most architectures)
-  // unless n were the same on all threads in a thread group.
-  // Assuming n is the same on all threads, any kind of unrolling is
-  // safe.  But currently llvm's notion of convergence isn't powerful
-  // enough to express this.
-  if (UCE.Convergent)
-    UP.AllowRemainder = false;
+  // TODO: This is somewhat conservative; we could allow the remainder if the
+  // trip count is uniform.
+  UP.AllowRemainder &= UCE.ConvergenceAllowsRuntime;
 
   // Try to find the trip count upper bound if we cannot find the exact trip
   // count.
@@ -1282,6 +1298,8 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
   if (!UP.Count)
     return LoopUnrollResult::Unmodified;
 
+  UP.Runtime &= UCE.ConvergenceAllowsRuntime;
+
   if (PP.PeelCount) {
     assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");
     LLVM_DEBUG(dbgs() << "PEELING loop %" << L->getHeader()->getName()
@@ -1324,11 +1342,16 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
 
   // Unroll the loop.
   Loop *RemainderLoop = nullptr;
+  UnrollLoopOptions ULO;
+  ULO.Count = UP.Count;
+  ULO.Force = UP.Force;
+  ULO.AllowExpensiveTripCount = UP.AllowExpensiveTripCount;
+  ULO.UnrollRemainder = UP.UnrollRemainder;
+  ULO.Runtime = UP.Runtime;
+  ULO.ForgetAllSCEV = ForgetAllSCEV;
+  ULO.Heart = getLoopConvergenceHeart(L);
   LoopUnrollResult UnrollResult = UnrollLoop(
-      L,
-      {UP.Count, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,
-       UP.UnrollRemainder, ForgetAllSCEV},
-      LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
+      L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
   if (UnrollResult == LoopUnrollResult::Unmodified)
     return LoopUnrollResult::Unmodified;
 
diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp
index c73d7c8d83bec1a582cfa05874d7466c764f9714..f36e21b296bd1fbf0a13949e7c672bc4f31aea3b 100644
--- a/llvm/lib/Transforms/Scalar/Reassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp
@@ -302,98 +302,7 @@ static BinaryOperator *LowerNegateToMultiply(Instruction *Neg) {
   return Res;
 }
 
-/// Returns k such that lambda(2^Bitwidth) = 2^k, where lambda is the Carmichael
-/// function. This means that x^(2^k) === 1 mod 2^Bitwidth for
-/// every odd x, i.e. x^(2^k) = 1 for every odd x in Bitwidth-bit arithmetic.
-/// Note that 0 <= k < Bitwidth, and if Bitwidth > 3 then x^(2^k) = 0 for every
-/// even x in Bitwidth-bit arithmetic.
-static unsigned CarmichaelShift(unsigned Bitwidth) {
-  if (Bitwidth < 3)
-    return Bitwidth - 1;
-  return Bitwidth - 2;
-}
-
-/// Add the extra weight 'RHS' to the existing weight 'LHS',
-/// reducing the combined weight using any special properties of the operation.
-/// The existing weight LHS represents the computation X op X op ... op X where
-/// X occurs LHS times.  The combined weight represents  X op X op ... op X with
-/// X occurring LHS + RHS times.  If op is "Xor" for example then the combined
-/// operation is equivalent to X if LHS + RHS is odd, or 0 if LHS + RHS is even;
-/// the routine returns 1 in LHS in the first case, and 0 in LHS in the second.
-static void IncorporateWeight(APInt &LHS, const APInt &RHS, unsigned Opcode) {
-  // If we were working with infinite precision arithmetic then the combined
-  // weight would be LHS + RHS.  But we are using finite precision arithmetic,
-  // and the APInt sum LHS + RHS may not be correct if it wraps (it is correct
-  // for nilpotent operations and addition, but not for idempotent operations
-  // and multiplication), so it is important to correctly reduce the combined
-  // weight back into range if wrapping would be wrong.
-
-  // If RHS is zero then the weight didn't change.
-  if (RHS.isMinValue())
-    return;
-  // If LHS is zero then the combined weight is RHS.
-  if (LHS.isMinValue()) {
-    LHS = RHS;
-    return;
-  }
-  // From this point on we know that neither LHS nor RHS is zero.
-
-  if (Instruction::isIdempotent(Opcode)) {
-    // Idempotent means X op X === X, so any non-zero weight is equivalent to a
-    // weight of 1.  Keeping weights at zero or one also means that wrapping is
-    // not a problem.
-    assert(LHS == 1 && RHS == 1 && "Weights not reduced!");
-    return; // Return a weight of 1.
-  }
-  if (Instruction::isNilpotent(Opcode)) {
-    // Nilpotent means X op X === 0, so reduce weights modulo 2.
-    assert(LHS == 1 && RHS == 1 && "Weights not reduced!");
-    LHS = 0; // 1 + 1 === 0 modulo 2.
-    return;
-  }
-  if (Opcode == Instruction::Add || Opcode == Instruction::FAdd) {
-    // TODO: Reduce the weight by exploiting nsw/nuw?
-    LHS += RHS;
-    return;
-  }
-
-  assert((Opcode == Instruction::Mul || Opcode == Instruction::FMul) &&
-         "Unknown associative operation!");
-  unsigned Bitwidth = LHS.getBitWidth();
-  // If CM is the Carmichael number then a weight W satisfying W >= CM+Bitwidth
-  // can be replaced with W-CM.  That's because x^W=x^(W-CM) for every Bitwidth
-  // bit number x, since either x is odd in which case x^CM = 1, or x is even in
-  // which case both x^W and x^(W - CM) are zero.  By subtracting off multiples
-  // of CM like this weights can always be reduced to the range [0, CM+Bitwidth)
-  // which by a happy accident means that they can always be represented using
-  // Bitwidth bits.
-  // TODO: Reduce the weight by exploiting nsw/nuw?  (Could do much better than
-  // the Carmichael number).
-  if (Bitwidth > 3) {
-    /// CM - The value of Carmichael's lambda function.
-    APInt CM = APInt::getOneBitSet(Bitwidth, CarmichaelShift(Bitwidth));
-    // Any weight W >= Threshold can be replaced with W - CM.
-    APInt Threshold = CM + Bitwidth;
-    assert(LHS.ult(Threshold) && RHS.ult(Threshold) && "Weights not reduced!");
-    // For Bitwidth 4 or more the following sum does not overflow.
-    LHS += RHS;
-    while (LHS.uge(Threshold))
-      LHS -= CM;
-  } else {
-    // To avoid problems with overflow do everything the same as above but using
-    // a larger type.
-    unsigned CM = 1U << CarmichaelShift(Bitwidth);
-    unsigned Threshold = CM + Bitwidth;
-    assert(LHS.getZExtValue() < Threshold && RHS.getZExtValue() < Threshold &&
-           "Weights not reduced!");
-    unsigned Total = LHS.getZExtValue() + RHS.getZExtValue();
-    while (Total >= Threshold)
-      Total -= CM;
-    LHS = Total;
-  }
-}
-
-using RepeatedValue = std::pair<Value*, APInt>;
+using RepeatedValue = std::pair<Value *, uint64_t>;
 
 /// Given an associative binary expression, return the leaf
 /// nodes in Ops along with their weights (how many times the leaf occurs).  The
@@ -475,7 +384,6 @@ static bool LinearizeExprTree(Instruction *I,
   assert((isa<UnaryOperator>(I) || isa<BinaryOperator>(I)) &&
          "Expected a UnaryOperator or BinaryOperator!");
   LLVM_DEBUG(dbgs() << "LINEARIZE: " << *I << '\n');
-  unsigned Bitwidth = I->getType()->getScalarType()->getPrimitiveSizeInBits();
   unsigned Opcode = I->getOpcode();
   assert(I->isAssociative() && I->isCommutative() &&
          "Expected an associative and commutative operation!");
@@ -490,8 +398,8 @@ static bool LinearizeExprTree(Instruction *I,
   // with their weights, representing a certain number of paths to the operator.
   // If an operator occurs in the worklist multiple times then we found multiple
   // ways to get to it.
-  SmallVector<std::pair<Instruction*, APInt>, 8> Worklist; // (Op, Weight)
-  Worklist.push_back(std::make_pair(I, APInt(Bitwidth, 1)));
+  SmallVector<std::pair<Instruction *, uint64_t>, 8> Worklist; // (Op, Weight)
+  Worklist.push_back(std::make_pair(I, 1));
   bool Changed = false;
 
   // Leaves of the expression are values that either aren't the right kind of
@@ -509,7 +417,7 @@ static bool LinearizeExprTree(Instruction *I,
 
   // Leaves - Keeps track of the set of putative leaves as well as the number of
   // paths to each leaf seen so far.
-  using LeafMap = DenseMap<Value *, APInt>;
+  using LeafMap = DenseMap<Value *, uint64_t>;
   LeafMap Leaves; // Leaf -> Total weight so far.
   SmallVector<Value *, 8> LeafOrder; // Ensure deterministic leaf output order.
   const DataLayout DL = I->getModule()->getDataLayout();
@@ -518,8 +426,8 @@ static bool LinearizeExprTree(Instruction *I,
   SmallPtrSet<Value *, 8> Visited; // For checking the iteration scheme.
 #endif
   while (!Worklist.empty()) {
-    std::pair<Instruction*, APInt> P = Worklist.pop_back_val();
-    I = P.first; // We examine the operands of this binary operator.
+    // We examine the operands of this binary operator.
+    auto [I, Weight] = Worklist.pop_back_val();
 
     if (isa<OverflowingBinaryOperator>(I)) {
       Flags.HasNUW &= I->hasNoUnsignedWrap();
@@ -528,7 +436,6 @@ static bool LinearizeExprTree(Instruction *I,
 
     for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) { // Visit operands.
       Value *Op = I->getOperand(OpIdx);
-      APInt Weight = P.second; // Number of paths to this operand.
       LLVM_DEBUG(dbgs() << "OPERAND: " << *Op << " (" << Weight << ")\n");
       assert(!Op->use_empty() && "No uses, so how did we get to it?!");
 
@@ -562,7 +469,8 @@ static bool LinearizeExprTree(Instruction *I,
                "In leaf map but not visited!");
 
         // Update the number of paths to the leaf.
-        IncorporateWeight(It->second, Weight, Opcode);
+        It->second += Weight;
+        assert(It->second >= Weight && "Weight overflows");
 
         // If we still have uses that are not accounted for by the expression
         // then it is not safe to modify the value.
@@ -625,10 +533,7 @@ static bool LinearizeExprTree(Instruction *I,
       // Node initially thought to be a leaf wasn't.
       continue;
     assert(!isReassociableOp(V, Opcode) && "Shouldn't be a leaf!");
-    APInt Weight = It->second;
-    if (Weight.isMinValue())
-      // Leaf already output or weight reduction eliminated it.
-      continue;
+    uint64_t Weight = It->second;
     // Ensure the leaf is only output once.
     It->second = 0;
     Ops.push_back(std::make_pair(V, Weight));
@@ -642,7 +547,7 @@ static bool LinearizeExprTree(Instruction *I,
   if (Ops.empty()) {
     Constant *Identity = ConstantExpr::getBinOpIdentity(Opcode, I->getType());
     assert(Identity && "Associative operation without identity!");
-    Ops.emplace_back(Identity, APInt(Bitwidth, 1));
+    Ops.emplace_back(Identity, 1);
   }
 
   return Changed;
@@ -1188,8 +1093,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) {
   Factors.reserve(Tree.size());
   for (unsigned i = 0, e = Tree.size(); i != e; ++i) {
     RepeatedValue E = Tree[i];
-    Factors.append(E.second.getZExtValue(),
-                   ValueEntry(getRank(E.first), E.first));
+    Factors.append(E.second, ValueEntry(getRank(E.first), E.first));
   }
 
   bool FoundFactor = false;
@@ -2368,7 +2272,7 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) {
   SmallVector<ValueEntry, 8> Ops;
   Ops.reserve(Tree.size());
   for (const RepeatedValue &E : Tree)
-    Ops.append(E.second.getZExtValue(), ValueEntry(getRank(E.first), E.first));
+    Ops.append(E.second, ValueEntry(getRank(E.first), E.first));
 
   LLVM_DEBUG(dbgs() << "RAIn:\t"; PrintOps(I, Ops); dbgs() << '\n');
 
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index eb471b259c7d4eab29f075e08addd91a306bae06..cfe63496a10074d691ba41c1ed42ea524c1abe56 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1221,7 +1221,6 @@ static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap,
       SmallPtrSet<const Value *, 4> ObjSet;
       SmallVector<Metadata *, 4> Scopes, NoAliases;
 
-      SmallSetVector<const Argument *, 4> NAPtrArgs;
       for (const Value *V : PtrArgs) {
         SmallVector<const Value *, 4> Objects;
         getUnderlyingObjects(V, Objects, /* LI = */ nullptr);
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 08ba65d9483e09763e6b38623ef66058c50e3475..3d950b151cd32fd87933191eab4d4c8930e826da 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -460,7 +460,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
                    L->dump());
         return Rotated;
       }
-      if (Metrics.convergent) {
+      if (Metrics.Convergence != ConvergenceKind::None) {
         LLVM_DEBUG(dbgs() << "LoopRotation: NOT rotating - contains convergent "
                    "instructions: ";
                    L->dump());
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 1216538195fbdf41d954dc7b9b52d367b3b386f2..90d7b99e9d817b9b99ac4205e3bfb5a91be6d007 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -419,6 +419,26 @@ void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
   }
 }
 
+// Loops containing convergent instructions that are uncontrolled or controlled
+// from outside the loop must have a count that divides their TripMultiple.
+LLVM_ATTRIBUTE_USED
+static bool canHaveUnrollRemainder(const Loop *L) {
+  if (getLoopConvergenceHeart(L))
+    return false;
+
+  // Check for uncontrolled convergent operations.
+  for (auto &BB : L->blocks()) {
+    for (auto &I : *BB) {
+      if (isa<ConvergenceControlInst>(I))
+        return true;
+      if (auto *CB = dyn_cast<CallBase>(&I))
+        if (CB->isConvergent())
+          return CB->getConvergenceControlToken();
+    }
+  }
+  return true;
+}
+
 /// Unroll the given loop by Count. The loop must be in LCSSA form.  Unrolling
 /// can only fail when the loop's latch block is not terminated by a conditional
 /// branch instruction. However, if the trip count (and multiple) are not known,
@@ -564,19 +584,8 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
     return LoopUnrollResult::Unmodified;
   }
 
-  // Loops containing convergent instructions cannot use runtime unrolling,
-  // as the prologue/epilogue may add additional control-dependencies to
-  // convergent operations.
-  LLVM_DEBUG(
-      {
-        bool HasConvergent = false;
-        for (auto &BB : L->blocks())
-          for (auto &I : *BB)
-            if (auto *CB = dyn_cast<CallBase>(&I))
-              HasConvergent |= CB->isConvergent();
-        assert((!HasConvergent || !ULO.Runtime) &&
-               "Can't runtime unroll if loop contains a convergent operation.");
-      });
+  assert((!ULO.Runtime || canHaveUnrollRemainder(L)) &&
+         "Can't runtime unroll if loop contains a convergent operation.");
 
   bool EpilogProfitability =
       UnrollRuntimeEpilog.getNumOccurrences() ? UnrollRuntimeEpilog
@@ -722,7 +731,7 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
       if (OldLoop)
         LoopsToSimplify.insert(NewLoops[OldLoop]);
 
-      if (*BB == Header)
+      if (*BB == Header) {
         // Loop over all of the PHI nodes in the block, changing them to use
         // the incoming values from the previous block.
         for (PHINode *OrigPHI : OrigPHINode) {
@@ -735,6 +744,16 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
           NewPHI->eraseFromParent();
         }
 
+        // Eliminate copies of the loop heart intrinsic, if any.
+        if (ULO.Heart) {
+          auto it = VMap.find(ULO.Heart);
+          assert(it != VMap.end());
+          Instruction *heartCopy = cast<Instruction>(it->second);
+          heartCopy->eraseFromParent();
+          VMap.erase(it);
+        }
+      }
+
       // Update our running map of newest clones
       LastValueMap[*BB] = New;
       for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index e1af02829c1da6fde2df5eccc613daad189c71a4..dd7150bc63ec4facf5218d5426f803b9f610dace 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -1016,12 +1016,17 @@ bool llvm::UnrollRuntimeLoopRemainder(
   auto UnrollResult = LoopUnrollResult::Unmodified;
   if (remainderLoop && UnrollRemainder) {
     LLVM_DEBUG(dbgs() << "Unrolling remainder loop\n");
-    UnrollResult =
-        UnrollLoop(remainderLoop,
-                   {/*Count*/ Count - 1, /*Force*/ false, /*Runtime*/ false,
-                    /*AllowExpensiveTripCount*/ false,
-                    /*UnrollRemainder*/ false, ForgetAllSCEV},
-                   LI, SE, DT, AC, TTI, /*ORE*/ nullptr, PreserveLCSSA);
+    UnrollLoopOptions ULO;
+    ULO.Count = Count - 1;
+    ULO.Force = false;
+    ULO.Runtime = false;
+    ULO.AllowExpensiveTripCount = false;
+    ULO.UnrollRemainder = false;
+    ULO.ForgetAllSCEV = ForgetAllSCEV;
+    assert(!getLoopConvergenceHeart(L) &&
+           "A loop with a convergence heart does not allow runtime unrolling.");
+    UnrollResult = UnrollLoop(remainderLoop, ULO, LI, SE, DT, AC, TTI,
+                              /*ORE*/ nullptr, PreserveLCSSA);
   }
 
   if (ResultLoop && UnrollResult != LoopUnrollResult::FullyUnrolled)
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index c437a44dda8d30467186796b96050871c4d3c6c8..09e6739fa9533cedb4ad84b4a9336d424c8ab90d 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -28,7 +28,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
 #define SCEV_DEBUG_WITH_TYPE(TYPE, X) DEBUG_WITH_TYPE(TYPE, X)
 #else
 #define SCEV_DEBUG_WITH_TYPE(TYPE, X)
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index fe6ec8819ff995751d9041e3b098ac5020dfbf6b..292739b6c5fdab9c9737681a0a7f1bf9910dcfdb 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6743,8 +6743,25 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     TableSize =
         (MaxCaseVal->getValue() - MinCaseVal->getValue()).getLimitedValue() + 1;
 
+  // If the default destination is unreachable, or if the lookup table covers
+  // all values of the conditional variable, branch directly to the lookup table
+  // BB. Otherwise, check that the condition is within the case range.
+  bool DefaultIsReachable = !SI->defaultDestUndefined();
+
   bool TableHasHoles = (NumResults < TableSize);
-  bool NeedMask = (TableHasHoles && !HasDefaultResults);
+
+  // If the table has holes but the default destination doesn't produce any
+  // constant results, the lookup table entries corresponding to the holes will
+  // contain undefined values.
+  bool AllHolesAreUndefined = TableHasHoles && !HasDefaultResults;
+
+  // If the default destination doesn't produce a constant result but is still
+  // reachable, and the lookup table has holes, we need to use a mask to
+  // determine if the current index should load from the lookup table or jump
+  // to the default case.
+  // The mask is unnecessary if the table has holes but the default destination
+  // is unreachable, as in that case the holes must also be unreachable.
+  bool NeedMask = AllHolesAreUndefined && DefaultIsReachable;
   if (NeedMask) {
     // As an extra penalty for the validity test we require more cases.
     if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark).
@@ -6766,12 +6783,6 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
          "It is impossible for a switch to have more entries than the max "
          "representable value of its input integer type's size.");
 
-  // If the default destination is unreachable, or if the lookup table covers
-  // all values of the conditional variable, branch directly to the lookup table
-  // BB. Otherwise, check that the condition is within the case range.
-  bool DefaultIsReachable =
-      !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
-
   // Create the BB that does the lookups.
   Module &Mod = *CommonDest->getParent()->getParent();
   BasicBlock *LookupBB = BasicBlock::Create(
@@ -6895,8 +6906,9 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
   for (PHINode *PHI : PHIs) {
     const ResultListTy &ResultList = ResultLists[PHI];
 
-    // If using a bitmask, use any value to fill the lookup table holes.
-    Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI];
+    // Use any value to fill the lookup table holes.
+    Constant *DV =
+        AllHolesAreUndefined ? ResultLists[PHI][0].second : DefaultResults[PHI];
     StringRef FuncName = Fn->getName();
     SwitchLookupTable Table(Mod, TableSize, TableIndexOffset, ResultList, DV,
                             DL, FuncName);
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 9674094024b9ec75a69d41c360e35f10dfdcf740..4caec07c5ac43770eb6ab24282ee5aa57d18d748 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_llvm_component_library(LLVMVectorize
   LoadStoreVectorizer.cpp
+  LoopIdiomVectorize.cpp
   LoopVectorizationLegality.cpp
   LoopVectorize.cpp
   SLPVectorizer.cpp
diff --git a/llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
similarity index 73%
rename from llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.cpp
rename to llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 8ae3f014d45e07a8b5e2ba47c7b24fb2bb54b643..38095b1433ebe341b2869c7b8cec062709189a1f 100644
--- a/llvm/lib/Target/AArch64/AArch64LoopIdiomTransform.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -1,4 +1,4 @@
-//===- AArch64LoopIdiomTransform.cpp - Loop idiom recognition -------------===//
+//===-------- LoopIdiomVectorize.cpp - Loop idiom vectorization -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -26,6 +26,10 @@
 //
 //===----------------------------------------------------------------------===//
 //
+// NOTE: This Pass matches a really specific loop pattern because it's only
+// supposed to be a temporary solution until our LoopVectorizer is powerful
+// enought to vectorize it automatically.
+//
 // TODO List:
 //
 // * Add support for the inverse case where we scan for a matching element.
@@ -35,7 +39,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "AArch64LoopIdiomTransform.h"
+#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -44,37 +48,30 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PatternMatch.h"
-#include "llvm/InitializePasses.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 
 using namespace llvm;
 using namespace PatternMatch;
 
-#define DEBUG_TYPE "aarch64-loop-idiom-transform"
-
-static cl::opt<bool>
-    DisableAll("disable-aarch64-lit-all", cl::Hidden, cl::init(false),
-               cl::desc("Disable AArch64 Loop Idiom Transform Pass."));
-
-static cl::opt<bool> DisableByteCmp(
-    "disable-aarch64-lit-bytecmp", cl::Hidden, cl::init(false),
-    cl::desc("Proceed with AArch64 Loop Idiom Transform Pass, but do "
-             "not convert byte-compare loop(s)."));
+#define DEBUG_TYPE "loop-idiom-vectorize"
 
-static cl::opt<bool> VerifyLoops(
-    "aarch64-lit-verify", cl::Hidden, cl::init(false),
-    cl::desc("Verify loops generated AArch64 Loop Idiom Transform Pass."));
+static cl::opt<bool> DisableAll("disable-loop-idiom-vectorize-all", cl::Hidden,
+                                cl::init(false),
+                                cl::desc("Disable Loop Idiom Vectorize Pass."));
 
-namespace llvm {
-
-void initializeAArch64LoopIdiomTransformLegacyPassPass(PassRegistry &);
-Pass *createAArch64LoopIdiomTransformPass();
+static cl::opt<bool>
+    DisableByteCmp("disable-loop-idiom-vectorize-bytecmp", cl::Hidden,
+                   cl::init(false),
+                   cl::desc("Proceed with Loop Idiom Vectorize Pass, but do "
+                            "not convert byte-compare loop(s)."));
 
-} // end namespace llvm
+static cl::opt<bool>
+    VerifyLoops("loop-idiom-vectorize-verify", cl::Hidden, cl::init(false),
+                cl::desc("Verify loops generated Loop Idiom Vectorize Pass."));
 
 namespace {
 
-class AArch64LoopIdiomTransform {
+class LoopIdiomVectorize {
   Loop *CurLoop = nullptr;
   DominatorTree *DT;
   LoopInfo *LI;
@@ -82,9 +79,9 @@ class AArch64LoopIdiomTransform {
   const DataLayout *DL;
 
 public:
-  explicit AArch64LoopIdiomTransform(DominatorTree *DT, LoopInfo *LI,
-                                     const TargetTransformInfo *TTI,
-                                     const DataLayout *DL)
+  explicit LoopIdiomVectorize(DominatorTree *DT, LoopInfo *LI,
+                              const TargetTransformInfo *TTI,
+                              const DataLayout *DL)
       : DT(DT), LI(LI), TTI(TTI), DL(DL) {}
 
   bool run(Loop *L);
@@ -107,74 +104,17 @@ private:
                             BasicBlock *EndBB);
   /// @}
 };
+} // anonymous namespace
 
-class AArch64LoopIdiomTransformLegacyPass : public LoopPass {
-public:
-  static char ID;
-
-  explicit AArch64LoopIdiomTransformLegacyPass() : LoopPass(ID) {
-    initializeAArch64LoopIdiomTransformLegacyPassPass(
-        *PassRegistry::getPassRegistry());
-  }
-
-  StringRef getPassName() const override {
-    return "Transform AArch64-specific loop idioms";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<LoopInfoWrapperPass>();
-    AU.addRequired<DominatorTreeWrapperPass>();
-    AU.addRequired<TargetTransformInfoWrapperPass>();
-  }
-
-  bool runOnLoop(Loop *L, LPPassManager &LPM) override;
-};
-
-bool AArch64LoopIdiomTransformLegacyPass::runOnLoop(Loop *L,
-                                                    LPPassManager &LPM) {
-
-  if (skipLoop(L))
-    return false;
-
-  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-  auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
-      *L->getHeader()->getParent());
-  return AArch64LoopIdiomTransform(
-             DT, LI, &TTI, &L->getHeader()->getModule()->getDataLayout())
-      .run(L);
-}
-
-} // end anonymous namespace
-
-char AArch64LoopIdiomTransformLegacyPass::ID = 0;
-
-INITIALIZE_PASS_BEGIN(
-    AArch64LoopIdiomTransformLegacyPass, "aarch64-lit",
-    "Transform specific loop idioms into optimized vector forms", false, false)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
-INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
-INITIALIZE_PASS_END(
-    AArch64LoopIdiomTransformLegacyPass, "aarch64-lit",
-    "Transform specific loop idioms into optimized vector forms", false, false)
-
-Pass *llvm::createAArch64LoopIdiomTransformPass() {
-  return new AArch64LoopIdiomTransformLegacyPass();
-}
-
-PreservedAnalyses
-AArch64LoopIdiomTransformPass::run(Loop &L, LoopAnalysisManager &AM,
-                                   LoopStandardAnalysisResults &AR,
-                                   LPMUpdater &) {
+PreservedAnalyses LoopIdiomVectorizePass::run(Loop &L, LoopAnalysisManager &AM,
+                                              LoopStandardAnalysisResults &AR,
+                                              LPMUpdater &) {
   if (DisableAll)
     return PreservedAnalyses::all();
 
   const auto *DL = &L.getHeader()->getModule()->getDataLayout();
 
-  AArch64LoopIdiomTransform LIT(&AR.DT, &AR.LI, &AR.TTI, DL);
+  LoopIdiomVectorize LIT(&AR.DT, &AR.LI, &AR.TTI, DL);
   if (!LIT.run(&L))
     return PreservedAnalyses::all();
 
@@ -183,11 +123,11 @@ AArch64LoopIdiomTransformPass::run(Loop &L, LoopAnalysisManager &AM,
 
 //===----------------------------------------------------------------------===//
 //
-//          Implementation of AArch64LoopIdiomTransform
+//          Implementation of LoopIdiomVectorize
 //
 //===----------------------------------------------------------------------===//
 
-bool AArch64LoopIdiomTransform::run(Loop *L) {
+bool LoopIdiomVectorize::run(Loop *L) {
   CurLoop = L;
 
   Function &F = *L->getHeader()->getParent();
@@ -211,7 +151,7 @@ bool AArch64LoopIdiomTransform::run(Loop *L) {
   return recognizeByteCompare();
 }
 
-bool AArch64LoopIdiomTransform::recognizeByteCompare() {
+bool LoopIdiomVectorize::recognizeByteCompare() {
   // Currently the transformation only works on scalable vector types, although
   // there is no fundamental reason why it cannot be made to work for fixed
   // width too.
@@ -224,7 +164,7 @@ bool AArch64LoopIdiomTransform::recognizeByteCompare() {
 
   BasicBlock *Header = CurLoop->getHeader();
 
-  // In AArch64LoopIdiomTransform::run we have already checked that the loop
+  // In LoopIdiomVectorize::run we have already checked that the loop
   // has a preheader so we can assume it's in a canonical form.
   if (CurLoop->getNumBackEdges() != 1 || CurLoop->getNumBlocks() != 2)
     return false;
@@ -242,8 +182,7 @@ bool AArch64LoopIdiomTransform::recognizeByteCompare() {
   //   %cmp.not = icmp eq i32 %inc, %n
   //   br i1 %cmp.not, label %while.end, label %while.body
   //
-  auto CondBBInsts = LoopBlocks[0]->instructionsWithoutDebug();
-  if (std::distance(CondBBInsts.begin(), CondBBInsts.end()) > 4)
+  if (LoopBlocks[0]->sizeWithoutDebug() > 4)
     return false;
 
   // The second block should contain 7 instructions, e.g.
@@ -257,8 +196,7 @@ bool AArch64LoopIdiomTransform::recognizeByteCompare() {
   //   %cmp.not.ld = icmp eq i8 %load.a, %load.b
   //   br i1 %cmp.not.ld, label %while.cond, label %while.end
   //
-  auto LoopBBInsts = LoopBlocks[1]->instructionsWithoutDebug();
-  if (std::distance(LoopBBInsts.begin(), LoopBBInsts.end()) > 7)
+  if (LoopBlocks[1]->sizeWithoutDebug() > 7)
     return false;
 
   // The incoming value to the PHI node from the loop should be an add of 1.
@@ -393,7 +331,7 @@ bool AArch64LoopIdiomTransform::recognizeByteCompare() {
   return true;
 }
 
-Value *AArch64LoopIdiomTransform::expandFindMismatch(
+Value *LoopIdiomVectorize::expandFindMismatch(
     IRBuilder<> &Builder, DomTreeUpdater &DTU, GetElementPtrInst *GEPA,
     GetElementPtrInst *GEPB, Instruction *Index, Value *Start, Value *MaxLen) {
   Value *PtrA = GEPA->getPointerOperand();
@@ -414,10 +352,10 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   //  1. A block for checking the zero-extended length exceeds 0
   //  2. A block to check that the start and end addresses of a given array
   //     lie on the same page.
-  //  3. The SVE loop preheader.
-  //  4. The first SVE loop block.
-  //  5. The SVE loop increment block.
-  //  6. A block we can jump to from the SVE loop when a mismatch is found.
+  //  3. The vector loop preheader.
+  //  4. The first vector loop block.
+  //  5. The vector loop increment block.
+  //  6. A block we can jump to from the vector loop when a mismatch is found.
   //  7. The first block of the scalar loop itself, containing PHIs , loads
   //  and cmp.
   //  8. A scalar loop increment block to increment the PHIs and go back
@@ -432,17 +370,17 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   BasicBlock *MemCheckBlock = BasicBlock::Create(
       Ctx, "mismatch_mem_check", EndBlock->getParent(), EndBlock);
 
-  BasicBlock *SVELoopPreheaderBlock = BasicBlock::Create(
-      Ctx, "mismatch_sve_loop_preheader", EndBlock->getParent(), EndBlock);
+  BasicBlock *VectorLoopPreheaderBlock = BasicBlock::Create(
+      Ctx, "mismatch_vec_loop_preheader", EndBlock->getParent(), EndBlock);
 
-  BasicBlock *SVELoopStartBlock = BasicBlock::Create(
-      Ctx, "mismatch_sve_loop", EndBlock->getParent(), EndBlock);
+  BasicBlock *VectorLoopStartBlock = BasicBlock::Create(
+      Ctx, "mismatch_vec_loop", EndBlock->getParent(), EndBlock);
 
-  BasicBlock *SVELoopIncBlock = BasicBlock::Create(
-      Ctx, "mismatch_sve_loop_inc", EndBlock->getParent(), EndBlock);
+  BasicBlock *VectorLoopIncBlock = BasicBlock::Create(
+      Ctx, "mismatch_vec_loop_inc", EndBlock->getParent(), EndBlock);
 
-  BasicBlock *SVELoopMismatchBlock = BasicBlock::Create(
-      Ctx, "mismatch_sve_loop_found", EndBlock->getParent(), EndBlock);
+  BasicBlock *VectorLoopMismatchBlock = BasicBlock::Create(
+      Ctx, "mismatch_vec_loop_found", EndBlock->getParent(), EndBlock);
 
   BasicBlock *LoopPreHeaderBlock = BasicBlock::Create(
       Ctx, "mismatch_loop_pre", EndBlock->getParent(), EndBlock);
@@ -456,26 +394,27 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   DTU.applyUpdates({{DominatorTree::Insert, Preheader, MinItCheckBlock},
                     {DominatorTree::Delete, Preheader, EndBlock}});
 
-  // Update LoopInfo with the new SVE & scalar loops.
-  auto SVELoop = LI->AllocateLoop();
+  // Update LoopInfo with the new vector & scalar loops.
+  auto VectorLoop = LI->AllocateLoop();
   auto ScalarLoop = LI->AllocateLoop();
 
   if (CurLoop->getParentLoop()) {
     CurLoop->getParentLoop()->addBasicBlockToLoop(MinItCheckBlock, *LI);
     CurLoop->getParentLoop()->addBasicBlockToLoop(MemCheckBlock, *LI);
-    CurLoop->getParentLoop()->addBasicBlockToLoop(SVELoopPreheaderBlock, *LI);
-    CurLoop->getParentLoop()->addChildLoop(SVELoop);
-    CurLoop->getParentLoop()->addBasicBlockToLoop(SVELoopMismatchBlock, *LI);
+    CurLoop->getParentLoop()->addBasicBlockToLoop(VectorLoopPreheaderBlock,
+                                                  *LI);
+    CurLoop->getParentLoop()->addChildLoop(VectorLoop);
+    CurLoop->getParentLoop()->addBasicBlockToLoop(VectorLoopMismatchBlock, *LI);
     CurLoop->getParentLoop()->addBasicBlockToLoop(LoopPreHeaderBlock, *LI);
     CurLoop->getParentLoop()->addChildLoop(ScalarLoop);
   } else {
-    LI->addTopLevelLoop(SVELoop);
+    LI->addTopLevelLoop(VectorLoop);
     LI->addTopLevelLoop(ScalarLoop);
   }
 
   // Add the new basic blocks to their associated loops.
-  SVELoop->addBasicBlockToLoop(SVELoopStartBlock, *LI);
-  SVELoop->addBasicBlockToLoop(SVELoopIncBlock, *LI);
+  VectorLoop->addBasicBlockToLoop(VectorLoopStartBlock, *LI);
+  VectorLoop->addBasicBlockToLoop(VectorLoopIncBlock, *LI);
 
   ScalarLoop->addBasicBlockToLoop(LoopStartBlock, *LI);
   ScalarLoop->addBasicBlockToLoop(LoopIncBlock, *LI);
@@ -537,7 +476,7 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
 
   Value *CombinedPageCmp = Builder.CreateOr(LhsPageCmp, RhsPageCmp);
   BranchInst *CombinedPageCmpCmpBr = BranchInst::Create(
-      LoopPreHeaderBlock, SVELoopPreheaderBlock, CombinedPageCmp);
+      LoopPreHeaderBlock, VectorLoopPreheaderBlock, CombinedPageCmp);
   CombinedPageCmpCmpBr->setMetadata(
       LLVMContext::MD_prof, MDBuilder(CombinedPageCmpCmpBr->getContext())
                                 .createBranchWeights(10, 90));
@@ -545,12 +484,12 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
 
   DTU.applyUpdates(
       {{DominatorTree::Insert, MemCheckBlock, LoopPreHeaderBlock},
-       {DominatorTree::Insert, MemCheckBlock, SVELoopPreheaderBlock}});
+       {DominatorTree::Insert, MemCheckBlock, VectorLoopPreheaderBlock}});
 
-  // Set up the SVE loop preheader, i.e. calculate initial loop predicate,
+  // Set up the vector loop preheader, i.e. calculate initial loop predicate,
   // zero-extend MaxLen to 64-bits, determine the number of vector elements
   // processed in each iteration, etc.
-  Builder.SetInsertPoint(SVELoopPreheaderBlock);
+  Builder.SetInsertPoint(VectorLoopPreheaderBlock);
 
   // At this point we know two things must be true:
   //  1. Start <= End
@@ -570,88 +509,91 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   Value *PFalse = Builder.CreateVectorSplat(PredVTy->getElementCount(),
                                             Builder.getInt1(false));
 
-  BranchInst *JumpToSVELoop = BranchInst::Create(SVELoopStartBlock);
-  Builder.Insert(JumpToSVELoop);
+  BranchInst *JumpToVectorLoop = BranchInst::Create(VectorLoopStartBlock);
+  Builder.Insert(JumpToVectorLoop);
 
-  DTU.applyUpdates(
-      {{DominatorTree::Insert, SVELoopPreheaderBlock, SVELoopStartBlock}});
+  DTU.applyUpdates({{DominatorTree::Insert, VectorLoopPreheaderBlock,
+                     VectorLoopStartBlock}});
 
-  // Set up the first SVE loop block by creating the PHIs, doing the vector
+  // Set up the first vector loop block by creating the PHIs, doing the vector
   // loads and comparing the vectors.
-  Builder.SetInsertPoint(SVELoopStartBlock);
-  PHINode *LoopPred = Builder.CreatePHI(PredVTy, 2, "mismatch_sve_loop_pred");
-  LoopPred->addIncoming(InitialPred, SVELoopPreheaderBlock);
-  PHINode *SVEIndexPhi = Builder.CreatePHI(I64Type, 2, "mismatch_sve_index");
-  SVEIndexPhi->addIncoming(ExtStart, SVELoopPreheaderBlock);
-  Type *SVELoadType = ScalableVectorType::get(Builder.getInt8Ty(), 16);
-  Value *Passthru = ConstantInt::getNullValue(SVELoadType);
-
-  Value *SVELhsGep =
-      Builder.CreateGEP(LoadType, PtrA, SVEIndexPhi, "", GEPA->isInBounds());
-  Value *SVELhsLoad = Builder.CreateMaskedLoad(SVELoadType, SVELhsGep, Align(1),
-                                               LoopPred, Passthru);
-
-  Value *SVERhsGep =
-      Builder.CreateGEP(LoadType, PtrB, SVEIndexPhi, "", GEPB->isInBounds());
-  Value *SVERhsLoad = Builder.CreateMaskedLoad(SVELoadType, SVERhsGep, Align(1),
-                                               LoopPred, Passthru);
-
-  Value *SVEMatchCmp = Builder.CreateICmpNE(SVELhsLoad, SVERhsLoad);
-  SVEMatchCmp = Builder.CreateSelect(LoopPred, SVEMatchCmp, PFalse);
-  Value *SVEMatchHasActiveLanes = Builder.CreateOrReduce(SVEMatchCmp);
-  BranchInst *SVEEarlyExit = BranchInst::Create(
-      SVELoopMismatchBlock, SVELoopIncBlock, SVEMatchHasActiveLanes);
-  Builder.Insert(SVEEarlyExit);
+  Builder.SetInsertPoint(VectorLoopStartBlock);
+  PHINode *LoopPred = Builder.CreatePHI(PredVTy, 2, "mismatch_vec_loop_pred");
+  LoopPred->addIncoming(InitialPred, VectorLoopPreheaderBlock);
+  PHINode *VectorIndexPhi = Builder.CreatePHI(I64Type, 2, "mismatch_vec_index");
+  VectorIndexPhi->addIncoming(ExtStart, VectorLoopPreheaderBlock);
+  Type *VectorLoadType = ScalableVectorType::get(Builder.getInt8Ty(), 16);
+  Value *Passthru = ConstantInt::getNullValue(VectorLoadType);
+
+  Value *VectorLhsGep =
+      Builder.CreateGEP(LoadType, PtrA, VectorIndexPhi, "", GEPA->isInBounds());
+  Value *VectorLhsLoad = Builder.CreateMaskedLoad(VectorLoadType, VectorLhsGep,
+                                                  Align(1), LoopPred, Passthru);
+
+  Value *VectorRhsGep =
+      Builder.CreateGEP(LoadType, PtrB, VectorIndexPhi, "", GEPB->isInBounds());
+  Value *VectorRhsLoad = Builder.CreateMaskedLoad(VectorLoadType, VectorRhsGep,
+                                                  Align(1), LoopPred, Passthru);
+
+  Value *VectorMatchCmp = Builder.CreateICmpNE(VectorLhsLoad, VectorRhsLoad);
+  VectorMatchCmp = Builder.CreateSelect(LoopPred, VectorMatchCmp, PFalse);
+  Value *VectorMatchHasActiveLanes = Builder.CreateOrReduce(VectorMatchCmp);
+  BranchInst *VectorEarlyExit = BranchInst::Create(
+      VectorLoopMismatchBlock, VectorLoopIncBlock, VectorMatchHasActiveLanes);
+  Builder.Insert(VectorEarlyExit);
 
   DTU.applyUpdates(
-      {{DominatorTree::Insert, SVELoopStartBlock, SVELoopMismatchBlock},
-       {DominatorTree::Insert, SVELoopStartBlock, SVELoopIncBlock}});
+      {{DominatorTree::Insert, VectorLoopStartBlock, VectorLoopMismatchBlock},
+       {DominatorTree::Insert, VectorLoopStartBlock, VectorLoopIncBlock}});
 
   // Increment the index counter and calculate the predicate for the next
   // iteration of the loop. We branch back to the start of the loop if there
   // is at least one active lane.
-  Builder.SetInsertPoint(SVELoopIncBlock);
-  Value *NewSVEIndexPhi = Builder.CreateAdd(SVEIndexPhi, VecLen, "",
-                                            /*HasNUW=*/true, /*HasNSW=*/true);
-  SVEIndexPhi->addIncoming(NewSVEIndexPhi, SVELoopIncBlock);
+  Builder.SetInsertPoint(VectorLoopIncBlock);
+  Value *NewVectorIndexPhi =
+      Builder.CreateAdd(VectorIndexPhi, VecLen, "",
+                        /*HasNUW=*/true, /*HasNSW=*/true);
+  VectorIndexPhi->addIncoming(NewVectorIndexPhi, VectorLoopIncBlock);
   Value *NewPred =
       Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
-                              {PredVTy, I64Type}, {NewSVEIndexPhi, ExtEnd});
-  LoopPred->addIncoming(NewPred, SVELoopIncBlock);
+                              {PredVTy, I64Type}, {NewVectorIndexPhi, ExtEnd});
+  LoopPred->addIncoming(NewPred, VectorLoopIncBlock);
 
   Value *PredHasActiveLanes =
       Builder.CreateExtractElement(NewPred, uint64_t(0));
-  BranchInst *SVELoopBranchBack =
-      BranchInst::Create(SVELoopStartBlock, EndBlock, PredHasActiveLanes);
-  Builder.Insert(SVELoopBranchBack);
+  BranchInst *VectorLoopBranchBack =
+      BranchInst::Create(VectorLoopStartBlock, EndBlock, PredHasActiveLanes);
+  Builder.Insert(VectorLoopBranchBack);
 
-  DTU.applyUpdates({{DominatorTree::Insert, SVELoopIncBlock, SVELoopStartBlock},
-                    {DominatorTree::Insert, SVELoopIncBlock, EndBlock}});
+  DTU.applyUpdates(
+      {{DominatorTree::Insert, VectorLoopIncBlock, VectorLoopStartBlock},
+       {DominatorTree::Insert, VectorLoopIncBlock, EndBlock}});
 
   // If we found a mismatch then we need to calculate which lane in the vector
   // had a mismatch and add that on to the current loop index.
-  Builder.SetInsertPoint(SVELoopMismatchBlock);
-  PHINode *FoundPred = Builder.CreatePHI(PredVTy, 1, "mismatch_sve_found_pred");
-  FoundPred->addIncoming(SVEMatchCmp, SVELoopStartBlock);
+  Builder.SetInsertPoint(VectorLoopMismatchBlock);
+  PHINode *FoundPred = Builder.CreatePHI(PredVTy, 1, "mismatch_vec_found_pred");
+  FoundPred->addIncoming(VectorMatchCmp, VectorLoopStartBlock);
   PHINode *LastLoopPred =
-      Builder.CreatePHI(PredVTy, 1, "mismatch_sve_last_loop_pred");
-  LastLoopPred->addIncoming(LoopPred, SVELoopStartBlock);
-  PHINode *SVEFoundIndex =
-      Builder.CreatePHI(I64Type, 1, "mismatch_sve_found_index");
-  SVEFoundIndex->addIncoming(SVEIndexPhi, SVELoopStartBlock);
+      Builder.CreatePHI(PredVTy, 1, "mismatch_vec_last_loop_pred");
+  LastLoopPred->addIncoming(LoopPred, VectorLoopStartBlock);
+  PHINode *VectorFoundIndex =
+      Builder.CreatePHI(I64Type, 1, "mismatch_vec_found_index");
+  VectorFoundIndex->addIncoming(VectorIndexPhi, VectorLoopStartBlock);
 
   Value *PredMatchCmp = Builder.CreateAnd(LastLoopPred, FoundPred);
   Value *Ctz = Builder.CreateIntrinsic(
       Intrinsic::experimental_cttz_elts, {ResType, PredMatchCmp->getType()},
       {PredMatchCmp, /*ZeroIsPoison=*/Builder.getInt1(true)});
   Ctz = Builder.CreateZExt(Ctz, I64Type);
-  Value *SVELoopRes64 = Builder.CreateAdd(SVEFoundIndex, Ctz, "",
-                                          /*HasNUW=*/true, /*HasNSW=*/true);
-  Value *SVELoopRes = Builder.CreateTrunc(SVELoopRes64, ResType);
+  Value *VectorLoopRes64 = Builder.CreateAdd(VectorFoundIndex, Ctz, "",
+                                             /*HasNUW=*/true, /*HasNSW=*/true);
+  Value *VectorLoopRes = Builder.CreateTrunc(VectorLoopRes64, ResType);
 
   Builder.Insert(BranchInst::Create(EndBlock));
 
-  DTU.applyUpdates({{DominatorTree::Insert, SVELoopMismatchBlock, EndBlock}});
+  DTU.applyUpdates(
+      {{DominatorTree::Insert, VectorLoopMismatchBlock, EndBlock}});
 
   // Generate code for scalar loop.
   Builder.SetInsertPoint(LoopPreHeaderBlock);
@@ -701,22 +643,22 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   //  1. We didn't find a mismatch in the scalar loop, so we return MaxLen.
   //  2. We exitted the scalar loop early due to a mismatch and need to return
   //  the index that we found.
-  //  3. We didn't find a mismatch in the SVE loop, so we return MaxLen.
-  //  4. We exitted the SVE loop early due to a mismatch and need to return
+  //  3. We didn't find a mismatch in the vector loop, so we return MaxLen.
+  //  4. We exitted the vector loop early due to a mismatch and need to return
   //  the index that we found.
   Builder.SetInsertPoint(EndBlock, EndBlock->getFirstInsertionPt());
   PHINode *ResPhi = Builder.CreatePHI(ResType, 4, "mismatch_result");
   ResPhi->addIncoming(MaxLen, LoopIncBlock);
   ResPhi->addIncoming(IndexPhi, LoopStartBlock);
-  ResPhi->addIncoming(MaxLen, SVELoopIncBlock);
-  ResPhi->addIncoming(SVELoopRes, SVELoopMismatchBlock);
+  ResPhi->addIncoming(MaxLen, VectorLoopIncBlock);
+  ResPhi->addIncoming(VectorLoopRes, VectorLoopMismatchBlock);
 
   Value *FinalRes = Builder.CreateTrunc(ResPhi, ResType);
 
   if (VerifyLoops) {
     ScalarLoop->verifyLoop();
-    SVELoop->verifyLoop();
-    if (!SVELoop->isRecursivelyLCSSAForm(*DT, *LI))
+    VectorLoop->verifyLoop();
+    if (!VectorLoop->isRecursivelyLCSSAForm(*DT, *LI))
       report_fatal_error("Loops must remain in LCSSA form!");
     if (!ScalarLoop->isRecursivelyLCSSAForm(*DT, *LI))
       report_fatal_error("Loops must remain in LCSSA form!");
@@ -725,10 +667,12 @@ Value *AArch64LoopIdiomTransform::expandFindMismatch(
   return FinalRes;
 }
 
-void AArch64LoopIdiomTransform::transformByteCompare(
-    GetElementPtrInst *GEPA, GetElementPtrInst *GEPB, PHINode *IndPhi,
-    Value *MaxLen, Instruction *Index, Value *Start, bool IncIdx,
-    BasicBlock *FoundBB, BasicBlock *EndBB) {
+void LoopIdiomVectorize::transformByteCompare(GetElementPtrInst *GEPA,
+                                              GetElementPtrInst *GEPB,
+                                              PHINode *IndPhi, Value *MaxLen,
+                                              Instruction *Index, Value *Start,
+                                              bool IncIdx, BasicBlock *FoundBB,
+                                              BasicBlock *EndBB) {
 
   // Insert the byte compare code at the end of the preheader block
   BasicBlock *Preheader = CurLoop->getLoopPreheader();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 943edc3520869f8bc3e0c5c0cbadd6d97a71c7d7..5bb88e4a57dc375ac1d3384ff94094f74929d367 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1325,20 +1325,7 @@ public:
   bool onlyFirstLaneUsed(const VPValue *Op) const override;
 
   /// Returns true if the recipe only uses the first part of operand \p Op.
-  bool onlyFirstPartUsed(const VPValue *Op) const override {
-    assert(is_contained(operands(), Op) &&
-           "Op must be an operand of the recipe");
-    if (getOperand(0) != Op)
-      return false;
-    switch (getOpcode()) {
-    default:
-      return false;
-    case VPInstruction::BranchOnCount:
-    case VPInstruction::CanonicalIVIncrementForPart:
-      return true;
-    };
-    llvm_unreachable("switch should return");
-  }
+  bool onlyFirstPartUsed(const VPValue *Op) const override;
 
   /// Returns true if this VPInstruction produces a scalar value from a vector,
   /// e.g. by performing a reduction or extracting a lane.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 05874688074351e50dfc7578bcb3e1761f7b5205..d6b4acb2bdba263ef3e935986f188b9ceb178e75 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -274,6 +274,13 @@ m_Mul(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
 }
 
+template <typename Op0_t, typename Op1_t>
+inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Mul,
+                             /* Commutative =*/true>
+m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
+  return m_Binary<Instruction::Mul, Op0_t, Op1_t, true>(Op0, Op1);
+}
+
 /// Match a binary OR operation. Note that while conceptually the operands can
 /// be matched commutatively, \p Commutative defaults to false in line with the
 /// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cb707d7c0e24f4542aeb849c0e53f23321f797ef..7a482455473e404488d9276c121ac2805b1abc7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -138,6 +138,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
     case VPInstruction::CalculateTripCountMinusVF:
     case VPInstruction::CanonicalIVIncrementForPart:
     case VPInstruction::ExtractFromEnd:
+    case VPInstruction::FirstOrderRecurrenceSplice:
     case VPInstruction::LogicalAnd:
     case VPInstruction::PtrAdd:
       return false;
@@ -324,9 +325,6 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
 
   if (Instruction::isBinaryOp(getOpcode())) {
     bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
-    if (Part != 0 && vputils::onlyFirstPartUsed(this))
-      return State.get(this, 0, OnlyFirstLaneUsed);
-
     Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed);
     Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed);
     auto *Res =
@@ -628,6 +626,7 @@ void VPInstruction::execute(VPTransformState &State) {
       canGenerateScalarForFirstLane() &&
       (vputils::onlyFirstLaneUsed(this) || isVectorToScalar());
   bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
+  bool OnlyFirstPartUsed = vputils::onlyFirstPartUsed(this);
   for (unsigned Part = 0; Part < State.UF; ++Part) {
     if (GeneratesPerAllLanes) {
       for (unsigned Lane = 0, NumLanes = State.VF.getKnownMinValue();
@@ -639,6 +638,13 @@ void VPInstruction::execute(VPTransformState &State) {
       continue;
     }
 
+    if (Part != 0 && OnlyFirstPartUsed && hasResult()) {
+      Value *Part0 = State.get(this, 0, /*IsScalar*/ GeneratesPerFirstLaneOnly);
+      State.set(this, Part0, Part,
+                /*IsScalar*/ GeneratesPerFirstLaneOnly);
+      continue;
+    }
+
     Value *GeneratedValue = generatePerPart(State, Part);
     if (!hasResult())
       continue;
@@ -674,6 +680,25 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   llvm_unreachable("switch should return");
 }
 
+bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
+  assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
+  if (Instruction::isBinaryOp(getOpcode()))
+    return vputils::onlyFirstPartUsed(this);
+
+  switch (getOpcode()) {
+  default:
+    return false;
+  case Instruction::ICmp:
+  case Instruction::Select:
+    return vputils::onlyFirstPartUsed(this);
+  case VPInstruction::BranchOnCount:
+  case VPInstruction::BranchOnCond:
+  case VPInstruction::CanonicalIVIncrementForPart:
+    return true;
+  };
+  llvm_unreachable("switch should return");
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPInstruction::dump() const {
   VPSlotTracker SlotTracker(getParent()->getPlan());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ab3b5cf2b9dabe66bcc6db2a0b7ebc8ecd5633bd..8ec67eb2f54bda7208dc4773f9d116d0fb12e914 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1037,8 +1037,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return;
   }
 
-  if (match(&R, m_CombineOr(m_Mul(m_VPValue(A), m_SpecificInt(1)),
-                            m_Mul(m_SpecificInt(1), m_VPValue(A)))))
+  if (match(&R, m_c_Mul(m_VPValue(A), m_SpecificInt(1))))
     return R.getVPSingleValue()->replaceAllUsesWith(A);
 }
 
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll
index eb55e6abc95e6e9ddec10db3d052c1458681efbc..ecf13320a5e81acb46aaa29c978e451bfa3209c0 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll
@@ -267,7 +267,228 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
+define void @ne_nsw_pos_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nsw_pos_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nsw_pos_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %pos_step = icmp sgt i32 %s, 0
+  call void @llvm.assume(i1 %pos_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nsw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nsw_neg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nsw_neg_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nsw_neg_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %neg_step = icmp slt i32 %s, 0
+  call void @llvm.assume(i1 %neg_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nsw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nsw_nonneg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nsw_nonneg_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nsw_nonneg_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %nonneg_step = icmp sge i32 %s, 0
+  call void @llvm.assume(i1 %nonneg_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nsw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nsw_unknown_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nsw_unknown_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nsw_unknown_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nsw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nuw_pos_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nuw_pos_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nuw_pos_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %pos_step = icmp sgt i32 %s, 0
+  call void @llvm.assume(i1 %pos_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nuw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nuw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nuw_neg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nuw_neg_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nuw_neg_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %neg_step = icmp slt i32 %s, 0
+  call void @llvm.assume(i1 %neg_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nuw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nuw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nuw_nonneg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nuw_nonneg_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nuw_nonneg_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %nonneg_step = icmp sge i32 %s, 0
+  call void @llvm.assume(i1 %nonneg_step)
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nuw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nuw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @ne_nuw_unknown_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress {
+;
+; CHECK-LABEL: 'ne_nuw_unknown_step'
+; CHECK-NEXT:  Determining loop execution counts for: @ne_nuw_unknown_step
+; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
+  %0 = load i32, ptr %arrayidx, align 4
+  %inc = add nuw i32 %0, 1
+  store i32 %inc, ptr %arrayidx, align 4
+  %add = add nuw i32 %i.05, %s
+  %cmp = icmp ne i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
 
+declare void @llvm.assume(i1)
 
 !8 = distinct !{!8, !9}
 !9 = !{!"llvm.loop.mustprogress"}
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.buffer.atomic.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.buffer.atomic.ll
index 3db2183845ff26fc1cbd60bc29a191241b83102a..6d0ca701e47a9eb8d07d2f30dff953f943a6222a 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.buffer.atomic.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.buffer.atomic.ll
@@ -1,93 +1,5 @@
 ; RUN: opt -mtriple amdgcn-mesa-mesa3d -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(
-define float @buffer_atomic_swap(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.add.i32(
-define float @buffer_atomic_add(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.sub.i32(
-define float @buffer_atomic_sub(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.sub.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.smin.i32(
-define float @buffer_atomic_smin(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.smin.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.umin.i32(
-define float @buffer_atomic_umin(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.umin.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.smax.i32(
-define float @buffer_atomic_smax(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.smax.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.umax.i32(
-define float @buffer_atomic_umax(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.umax.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.and.i32(
-define float @buffer_atomic_and(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.and.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.or.i32(
-define float @buffer_atomic_or(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.or.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.xor.i32(
-define float @buffer_atomic_xor(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.xor.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(
-define float @buffer_atomic_cmpswap(<4 x i32> inreg %rsrc, i32 inreg %data, i32 inreg %cmp) #0 {
-main_body:
-  %orig = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %data, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %r = bitcast i32 %orig to float
-  ret float %r
-}
-
 ;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.raw.buffer.atomic.swap.i32(
 define float @raw_buffer_atomic_swap(<4 x i32> inreg %rsrc, i32 inreg %data) #0 {
 main_body:
@@ -440,18 +352,6 @@ main_body:
   ret float %r
 }
 
-declare i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.sub.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.smin.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.umin.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.smax.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.umax.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.and.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.or.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.xor.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32, i32, <4 x i32>, i32, i32, i1) #0
-
 declare i32 @llvm.amdgcn.raw.buffer.atomic.swap.i32(i32, <4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.sub.i32(i32, <4 x i32>, i32, i32, i32) #0
diff --git a/llvm/test/Bindings/OCaml/core.ml b/llvm/test/Bindings/OCaml/core.ml
index 36f4c468d99f48ded5731adef2d48a90648dce20..923a3541e239c39cbad3d86dbff9109d870665b1 100644
--- a/llvm/test/Bindings/OCaml/core.ml
+++ b/llvm/test/Bindings/OCaml/core.ml
@@ -1150,7 +1150,7 @@ let test_builder () =
     (* CHECK: ret{{.*}}P1
      *)
     let ret = build_ret p1 atentry in
-    position_before ret atentry
+    position_before_dbg_records ret atentry
   end;
 
   (* see test/Feature/exception.ll *)
diff --git a/llvm/test/Bindings/llvm-c/debug_info.ll b/llvm/test/Bindings/llvm-c/debug_info.ll
index 9358bac59bd21e628964c3843804d83f6c39f407..71986fbb2348ff4c13eb852ccabd920a9f7faedb 100644
--- a/llvm/test/Bindings/llvm-c/debug_info.ll
+++ b/llvm/test/Bindings/llvm-c/debug_info.ll
@@ -10,7 +10,10 @@
 ; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !40, metadata !DIExpression()), !dbg !43
 ; CHECK-NEXT:   br label %vars
 ; CHECK:      vars:
+; CHECK-NEXT:   %p1 = phi i64 [ 0, %entry ]
+; CHECK-NEXT:   %p2 = phi i64 [ 0, %entry ]
 ; CHECK-NEXT:   call void @llvm.dbg.value(metadata i64 0, metadata !41, metadata !DIExpression(DW_OP_constu, 0, DW_OP_stack_value)), !dbg !44
+; CHECK-NEXT:   %a = add i64 %p1, %p2
 ; CHECK-NEXT:   ret i64 0
 ; CHECK-NEXT: }
 
diff --git a/llvm/test/Bindings/llvm-c/debug_info_new_format.ll b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
index 05b6ef4de9adbb44d5590bfc7065979f032f2f44..1b6f2c4ea403aabead79fe30d781079f7595fcbe 100644
--- a/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
+++ b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
@@ -11,7 +11,10 @@
 ; CHECK-NEXT:     #dbg_declare(i64 0, !40, !DIExpression(), !43)
 ; CHECK-NEXT:   br label %vars
 ; CHECK:      vars:
+; CHECK-NEXT:   %p1 = phi i64 [ 0, %entry ]
+; CHECK-NEXT:   %p2 = phi i64 [ 0, %entry ]
 ; CHECK-NEXT:     #dbg_value(i64 0, !41, !DIExpression(DW_OP_constu, 0, DW_OP_stack_value), !44)
+; CHECK-NEXT:   %a = add i64 %p1, %p2
 ; CHECK-NEXT:   ret i64 0
 ; CHECK-NEXT: }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index a61931b898aea57595e8c1b5b95b3f6f99f6e50e..eb94cc5d0fb610942457c0d960f4c48ceda47a7b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -2313,6 +2313,14 @@ define float @test_sin_f32(float %x) {
   ret float %y
 }
 
+declare float @llvm.tan.f32(float)
+define float @test_tan_f32(float %x) {
+  ; CHECK-LABEL: name:            test_tan_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FTAN %{{[0-9]+}}
+  %y = call float @llvm.tan.f32(float %x)
+  ret float %y
+}
+
 declare float @llvm.sqrt.f32(float)
 define float @test_sqrt_f32(float %x) {
   ; CHECK-LABEL: name:            test_sqrt_f32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9b7a44954afdb71bf0936287ffcfaa8bc79e3ef5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir
@@ -0,0 +1,113 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s | FileCheck %s
+
+...
+---
+name:            sum_of_vscale
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: sum_of_vscale
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %sum:_(s64) = G_VSCALE i64 20
+    ; CHECK-NEXT: $x0 = COPY %sum(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %rhs:_(s64) = G_VSCALE i64 11
+    %lhs:_(s64) = G_VSCALE i64 9
+    %sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64)
+    $x0 = COPY %sum(s64)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            sum_of_vscale_multi_use
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: sum_of_vscale_multi_use
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %rhs:_(s64) = G_VSCALE i64 11
+    ; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 9
+    ; CHECK-NEXT: %sum:_(s64) = nsw G_ADD %lhs, %rhs
+    ; CHECK-NEXT: $x0 = COPY %sum(s64)
+    ; CHECK-NEXT: $x1 = COPY %rhs(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %rhs:_(s64) = G_VSCALE i64 11
+    %lhs:_(s64) = G_VSCALE i64 9
+    %sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64)
+    $x0 = COPY %sum(s64)
+    $x1 = COPY %rhs(s64)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            mul_of_vscale
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: mul_of_vscale
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %mul:_(s64) = G_VSCALE i64 99
+    ; CHECK-NEXT: $x0 = COPY %mul(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %rhs:_(s64) = G_CONSTANT i64 11
+    %lhs:_(s64) = G_VSCALE i64 9
+    %mul:_(s64) = nsw G_MUL %lhs(s64), %rhs(s64)
+    $x0 = COPY %mul(s64)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            sub_of_vscale
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: sub_of_vscale
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[VSCALE:%[0-9]+]]:_(s64) = G_VSCALE i64 -9
+    ; CHECK-NEXT: %sub:_(s64) = nsw G_ADD %x, [[VSCALE]]
+    ; CHECK-NEXT: $x0 = COPY %sub(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %x:_(s64) = COPY $x0
+    %rhs:_(s64) = G_VSCALE i64 9
+    %sub:_(s64) = nsw G_SUB %x(s64), %rhs(s64)
+    $x0 = COPY %sub(s64)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            shl_of_vscale
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: shl_of_vscale
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %shl:_(s64) = G_VSCALE i64 44
+    ; CHECK-NEXT: $x0 = COPY %shl(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %rhs:_(s64) = G_CONSTANT i64 2
+    %lhs:_(s64) = G_VSCALE i64 11
+    %shl:_(s64) = nsw G_SHL %lhs(s64), %rhs(s64)
+    $x0 = COPY %shl(s64)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            shl_of_vscale_wrong_flag
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: shl_of_vscale_wrong_flag
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %rhs:_(s64) = G_CONSTANT i64 2
+    ; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 11
+    ; CHECK-NEXT: %shl:_(s64) = nuw G_SHL %lhs, %rhs(s64)
+    ; CHECK-NEXT: $x0 = COPY %shl(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %rhs:_(s64) = G_CONSTANT i64 2
+    %lhs:_(s64) = G_VSCALE i64 11
+    %shl:_(s64) = nuw G_SHL %lhs(s64), %rhs(s64)
+    $x0 = COPY %shl(s64)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir
new file mode 100644
index 0000000000000000000000000000000000000000..455aae0128364e8e682442be01e92e60a0a0e3fe
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.tan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.tan
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &tanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &tanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &tanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &tanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FTAN %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.tan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.tan
+
+    ; This is big, so let's just check for the 8 calls to tanf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: BL &tanf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FTAN %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.tan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.tan
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FTAN %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.tan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.tan
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &tanf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FTAN %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.tan
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.tan
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &tan
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &tan
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FTAN %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_tan_half
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_tan_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &tanf
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FTAN %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index d71111b57efe51b9af8135f125af97aa96df7340..04c9a08c500386d76fc2387386debd530db72e94 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -675,8 +675,9 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FTAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7cc5041d59f49e250f11952ca2e4609d93e11793
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
+
+define <4 x i32> @deinterleave_shuffle_v8i32(<8 x i32> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addp v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = add <4 x i32> %r0, %r1
+  ret <4 x i32> %o
+}
+
+define <4 x i32> @deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8i32_c:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addp v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = add <4 x i32> %r1, %r0
+  ret <4 x i32> %o
+}
+
+define <2 x i32> @deinterleave_shuffle_v4i32(<4 x i32> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    zip1 v2.2s, v0.2s, v1.2s
+; CHECK-NEXT:    zip2 v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    add v0.2s, v2.2s, v0.2s
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+  %r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+  %o = add <2 x i32> %r0, %r1
+  ret <2 x i32> %o
+}
+
+define <8 x i16> @deinterleave_shuffle_v16i16(<16 x i16> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addp v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %r1 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %o = add <8 x i16> %r0, %r1
+  ret <8 x i16> %o
+}
+
+define <16 x i8> @deinterleave_shuffle_v32i8(<32 x i8> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addp v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %r1 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %o = add <16 x i8> %r0, %r1
+  ret <16 x i8> %o
+}
+
+define <4 x i64> @deinterleave_shuffle_v8i64(<8 x i64> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addp v2.2d, v2.2d, v3.2d
+; CHECK-NEXT:    addp v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    mov v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = add <4 x i64> %r0, %r1
+  ret <4 x i64> %o
+}
+
+define <4 x float> @deinterleave_shuffle_v8f32(<8 x float> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddp v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = fadd <4 x float> %r0, %r1
+  ret <4 x float> %o
+}
+
+define <4 x float> @deinterleave_shuffle_v8f32_c(<8 x float> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8f32_c:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddp v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = fadd <4 x float> %r1, %r0
+  ret <4 x float> %o
+}
+
+define <8 x half> @deinterleave_shuffle_v16f16(<16 x half> %a) {
+; CHECK-NOFP16-LABEL: deinterleave_shuffle_v16f16:
+; CHECK-NOFP16:       // %bb.0:
+; CHECK-NOFP16-NEXT:    uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-NOFP16-NEXT:    fadd v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-NEXT:    fadd v2.4s, v2.4s, v0.4s
+; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: deinterleave_shuffle_v16f16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    faddp v0.8h, v0.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
+  %r0 = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %r1 = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %o = fadd <8 x half> %r0, %r1
+  ret <8 x half> %o
+}
+
+define <4 x double> @deinterleave_shuffle_v8f64(<8 x double> %a) {
+; CHECK-LABEL: deinterleave_shuffle_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddp v2.2d, v2.2d, v3.2d
+; CHECK-NEXT:    faddp v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    mov v1.16b, v2.16b
+; CHECK-NEXT:    ret
+  %r0 = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %r1 = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = fadd <4 x double> %r0, %r1
+  ret <4 x double> %o
+}
+
+define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: udot:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-NEXT:    ushll v4.8h, v2.8b, #0
+; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT:    umull2 v5.4s, v3.8h, v4.8h
+; CHECK-NEXT:    umull v3.4s, v3.4h, v4.4h
+; CHECK-NEXT:    umull2 v4.4s, v1.8h, v2.8h
+; CHECK-NEXT:    umull v1.4s, v1.4h, v2.4h
+; CHECK-NEXT:    addp v2.4s, v3.4s, v5.4s
+; CHECK-NEXT:    addp v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    addp v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %za = zext <16 x i8> %a to <16 x i32>
+  %zb = zext <16 x i8> %b to <16 x i32>
+  %m = mul <16 x i32> %za, %zb
+  %r0 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %r1 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %m2 = add <8 x i32> %r0, %r1
+  %s0 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s1 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = add <4 x i32> %s0, %s1
+  %n = add <4 x i32> %z, %o
+  ret <4 x i32> %n
+}
+
+define <4 x i32> @sdot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sdot:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-NEXT:    sshll v4.8h, v2.8b, #0
+; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    sshll2 v2.8h, v2.16b, #0
+; CHECK-NEXT:    smull2 v5.4s, v3.8h, v4.8h
+; CHECK-NEXT:    smull v3.4s, v3.4h, v4.4h
+; CHECK-NEXT:    smull2 v4.4s, v1.8h, v2.8h
+; CHECK-NEXT:    smull v1.4s, v1.4h, v2.4h
+; CHECK-NEXT:    addp v2.4s, v3.4s, v5.4s
+; CHECK-NEXT:    addp v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    addp v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %za = sext <16 x i8> %a to <16 x i32>
+  %zb = sext <16 x i8> %b to <16 x i32>
+  %m = mul <16 x i32> %za, %zb
+  %r0 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %r1 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %m2 = add <8 x i32> %r0, %r1
+  %s0 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %s1 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %o = add <4 x i32> %s0, %s1
+  %n = add <4 x i32> %z, %o
+  ret <4 x i32> %n
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll b/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll
index 3b6c4fa875e6045e0867dbfedf029417021fcd7b..dafdcf82f311d48cb82baf44b41808166beaf82f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll
@@ -12,7 +12,7 @@ entry:
 
 for.body:
 ; CHECK: for.body
-; CHECK: ldr w{{[0-9]+}}, [x{{[0-9]+}}, x{{[0-9]+}}]
+; CHECK: ldr w{{[0-9]+}}, [x{{[0-9]+}}]
 ; CHECK: add x[[REG:[0-9]+]],
 ; CHECK:                      x[[REG]], #1, lsl  #12
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
index 8c7b31fd34c488b5c13ddbd6d8152ce318514019..114203e46f196b29b3a823e1a054c7f6d02b5340 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll
@@ -176,13 +176,13 @@ exit:
 ; CHECK: ********** MI Scheduling **********
 ; CHECK: LDURDi_LDRDui:%bb.1 vector_body
 ;
-; CHECK: Cluster ld/st SU(2) - SU(6)
-; CHECK: Cluster ld/st SU(3) - SU(7)
+; CHECK: Cluster ld/st SU(0) - SU(4)
+; CHECK: Cluster ld/st SU(1) - SU(5)
 ;
-; CHECK: SU(2): %{{[0-9]+}}:fpr64 = LDURDi
-; CHECK: SU(3): %{{[0-9]+}}:fpr64 = LDURDi
-; CHECK: SU(6): %{{[0-9]+}}:fpr64 = LDRDui
-; CHECK: SU(7): %{{[0-9]+}}:fpr64 = LDRDui
+; CHECK: SU(0): %{{[0-9]+}}:fpr64 = LDURDi
+; CHECK: SU(1): %{{[0-9]+}}:fpr64 = LDURDi
+; CHECK: SU(4): %{{[0-9]+}}:fpr64 = LDRDui
+; CHECK: SU(5): %{{[0-9]+}}:fpr64 = LDRDui
 ;
 define void @LDURDi_LDRDui(ptr nocapture readonly %arg) {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/arm64-uzp.ll b/llvm/test/CodeGen/AArch64/arm64-uzp.ll
index bd6bf1bf1578412d80432a58442b797f98b865bb..02068428c3a4fa28ddaa3cefcb8c60f10dabb15d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-uzp.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-uzp.ll
@@ -33,11 +33,11 @@ define <16 x i8> @vuzpQi8(<16 x i8> %A, <16 x i8> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.16b v2, v0, v1
 ; CHECK-NEXT:    uzp2.16b v0, v0, v1
-; CHECK-NEXT:    add.16b v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <16 x i8> %A, <16 x i8> %B, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %tmp4 = shufflevector <16 x i8> %A, <16 x i8> %B, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-  %tmp5 = add <16 x i8> %tmp3, %tmp4
+  %tmp5 = xor <16 x i8> %tmp3, %tmp4
   ret <16 x i8> %tmp5
 }
 
@@ -46,11 +46,11 @@ define <8 x i16> @vuzpQi16(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.8h v2, v0, v1
 ; CHECK-NEXT:    uzp2.8h v0, v0, v1
-; CHECK-NEXT:    add.8h v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   ret <8 x i16> %tmp5
 }
 
@@ -59,11 +59,11 @@ define <4 x i32> @vuzpQi32(<4 x i32> %A, <4 x i32> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.4s v2, v0, v1
 ; CHECK-NEXT:    uzp2.4s v0, v0, v1
-; CHECK-NEXT:    add.4s v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %tmp4 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  %tmp5 = add <4 x i32> %tmp3, %tmp4
+  %tmp5 = xor <4 x i32> %tmp3, %tmp4
   ret <4 x i32> %tmp5
 }
 
@@ -72,11 +72,11 @@ define <4 x float> @vuzpQf(<4 x float> %A, <4 x float> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.4s v2, v0, v1
 ; CHECK-NEXT:    uzp2.4s v0, v0, v1
-; CHECK-NEXT:    fadd.4s v0, v2, v0
+; CHECK-NEXT:    fsub.4s v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <4 x float> %A, <4 x float> %B, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %tmp4 = shufflevector <4 x float> %A, <4 x float> %B, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  %tmp5 = fadd <4 x float> %tmp3, %tmp4
+  %tmp5 = fsub <4 x float> %tmp3, %tmp4
   ret <4 x float> %tmp5
 }
 
@@ -100,11 +100,11 @@ define <8 x i16> @vuzpQi16_undef1(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.8h v2, v0, v1
 ; CHECK-NEXT:    uzp2.8h v0, v0, v1
-; CHECK-NEXT:    add.8h v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 0, i32 undef, i32 4, i32 undef, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 1, i32 3, i32 5, i32 undef, i32 undef, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   ret <8 x i16> %tmp5
 }
 
@@ -113,11 +113,11 @@ define <8 x i16> @vuzpQi16_undef0(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.8h v2, v0, v1
 ; CHECK-NEXT:    uzp2.8h v0, v0, v1
-; CHECK-NEXT:    add.8h v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   ret <8 x i16> %tmp5
 }
 
@@ -126,11 +126,11 @@ define <8 x i16> @vuzpQi16_undef01(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.8h v2, v0, v1
 ; CHECK-NEXT:    uzp2.8h v0, v0, v1
-; CHECK-NEXT:    add.8h v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 undef, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 undef, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   ret <8 x i16> %tmp5
 }
 
@@ -139,10 +139,10 @@ define <8 x i16> @vuzpQi16_undef012(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uzp1.8h v2, v0, v1
 ; CHECK-NEXT:    uzp2.8h v0, v0, v1
-; CHECK-NEXT:    add.8h v0, v2, v0
+; CHECK-NEXT:    eor.16b v0, v2, v0
 ; CHECK-NEXT:    ret
   %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 6, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 7, i32 9, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   ret <8 x i16> %tmp5
 }
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
index ac2b21af29ab7b243b876eb5a15e7cdb99be2ed4..2ef35283568c3ca75d7e16a10aa42c38ade6bb8b 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll
@@ -15,36 +15,34 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
 ; CHECK-LABEL: complex_mul_v2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    mov w9, #100 // =0x64
-; CHECK-NEXT:    cntd x10
-; CHECK-NEXT:    whilelo p1.d, xzr, x9
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    mov w8, #100 // =0x64
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    whilelo p1.d, xzr, x8
+; CHECK-NEXT:    rdvl x10, #2
+; CHECK-NEXT:    mov x11, x9
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x12, x10
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
-; CHECK-NEXT:    add x13, x0, x8
-; CHECK-NEXT:    add x14, x1, x8
-; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
-; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x14, #1, mul vl]
-; CHECK-NEXT:    add x8, x8, x11
-; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x13]
-; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x14]
+; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
+; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x0]
+; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x1]
+; CHECK-NEXT:    add x1, x1, x10
+; CHECK-NEXT:    add x0, x0, x10
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #90
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
 ; CHECK-NEXT:    mov z0.d, p2/m, z7.d
 ; CHECK-NEXT:    mov z1.d, p1/m, z6.d
-; CHECK-NEXT:    whilelo p1.d, x12, x9
-; CHECK-NEXT:    add x12, x12, x10
+; CHECK-NEXT:    whilelo p1.d, x11, x8
+; CHECK-NEXT:    add x11, x11, x9
 ; CHECK-NEXT:    b.mi .LBB0_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
@@ -114,39 +112,37 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr %
 ; CHECK-LABEL: complex_mul_predicated_v2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    cntd x10
-; CHECK-NEXT:    mov w12, #100 // =0x64
-; CHECK-NEXT:    neg x11, x10
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    mov w11, #100 // =0x64
+; CHECK-NEXT:    neg x10, x9
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    mov x9, xzr
-; CHECK-NEXT:    and x11, x11, x12
-; CHECK-NEXT:    rdvl x12, #2
+; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    rdvl x11, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB1_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ld1w { z2.d }, p0/z, [x2, x9, lsl #2]
-; CHECK-NEXT:    add x13, x0, x8
-; CHECK-NEXT:    add x14, x1, x8
+; CHECK-NEXT:    ld1w { z2.d }, p0/z, [x2, x8, lsl #2]
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
-; CHECK-NEXT:    add x9, x9, x10
-; CHECK-NEXT:    add x8, x8, x12
-; CHECK-NEXT:    cmpne p2.d, p0/z, z2.d, #0
-; CHECK-NEXT:    cmp x11, x9
-; CHECK-NEXT:    zip2 p1.d, p2.d, p2.d
-; CHECK-NEXT:    zip1 p2.d, p2.d, p2.d
-; CHECK-NEXT:    ld1d { z2.d }, p1/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z4.d }, p1/z, [x14, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
-; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
+; CHECK-NEXT:    add x8, x8, x9
+; CHECK-NEXT:    cmpne p1.d, p0/z, z2.d, #0
+; CHECK-NEXT:    cmp x10, x8
+; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
+; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
+; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x0]
+; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x1]
+; CHECK-NEXT:    add x1, x1, x11
+; CHECK-NEXT:    add x0, x0, x11
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #90
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
-; CHECK-NEXT:    mov z0.d, p1/m, z7.d
-; CHECK-NEXT:    mov z1.d, p2/m, z6.d
+; CHECK-NEXT:    mov z0.d, p2/m, z7.d
+; CHECK-NEXT:    mov z1.d, p1/m, z6.d
 ; CHECK-NEXT:    b.ne .LBB1_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
@@ -218,38 +214,38 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt
 ; CHECK-LABEL: complex_mul_predicated_x2_v2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    mov w10, #100 // =0x64
+; CHECK-NEXT:    mov w8, #100 // =0x64
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    whilelo p1.d, xzr, x8
+; CHECK-NEXT:    rdvl x10, #2
+; CHECK-NEXT:    cnth x11
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    whilelo p1.d, xzr, x10
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    mov x9, xzr
-; CHECK-NEXT:    cntd x11
-; CHECK-NEXT:    rdvl x12, #2
+; CHECK-NEXT:    mov x12, x9
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB2_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ld1w { z2.d }, p1/z, [x2, x9, lsl #2]
-; CHECK-NEXT:    add x13, x0, x8
-; CHECK-NEXT:    add x14, x1, x8
+; CHECK-NEXT:    ld1w { z2.d }, p1/z, [x2]
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
-; CHECK-NEXT:    add x9, x9, x11
-; CHECK-NEXT:    add x8, x8, x12
-; CHECK-NEXT:    cmpne p2.d, p1/z, z2.d, #0
-; CHECK-NEXT:    zip2 p1.d, p2.d, p2.d
-; CHECK-NEXT:    zip1 p2.d, p2.d, p2.d
-; CHECK-NEXT:    ld1d { z2.d }, p1/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z4.d }, p1/z, [x14, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
-; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
+; CHECK-NEXT:    add x2, x2, x11
+; CHECK-NEXT:    cmpne p1.d, p1/z, z2.d, #0
+; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
+; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
+; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x0]
+; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x1]
+; CHECK-NEXT:    add x1, x1, x10
+; CHECK-NEXT:    add x0, x0, x10
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #90
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
-; CHECK-NEXT:    mov z0.d, p1/m, z7.d
-; CHECK-NEXT:    whilelo p1.d, x9, x10
-; CHECK-NEXT:    mov z1.d, p2/m, z6.d
+; CHECK-NEXT:    mov z0.d, p2/m, z7.d
+; CHECK-NEXT:    mov z1.d, p1/m, z6.d
+; CHECK-NEXT:    whilelo p1.d, x12, x8
+; CHECK-NEXT:    add x12, x12, x9
 ; CHECK-NEXT:    b.mi .LBB2_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
index af07519ad53daa37dae8a1d85d023913693d7c0b..8e26ef6b87eccac64bcaebf0e80dec08af6de90a 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
@@ -15,30 +15,27 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
 ; CHECK-LABEL: complex_mul_v2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    ptrue p1.b
-; CHECK-NEXT:    neg x10, x9
-; CHECK-NEXT:    mov w11, #100 // =0x64
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    mov w10, #100 // =0x64
+; CHECK-NEXT:    neg x9, x8
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    and x10, x10, x11
-; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    and x9, x9, x10
+; CHECK-NEXT:    rdvl x10, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x12, x0, x8
-; CHECK-NEXT:    add x13, x1, x8
-; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
-; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
-; CHECK-NEXT:    subs x10, x10, x9
-; CHECK-NEXT:    add x8, x8, x11
-; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #0
-; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #0
-; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #90
-; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #90
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT:    subs x9, x9, x8
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x1]
+; CHECK-NEXT:    add x1, x1, x10
+; CHECK-NEXT:    add x0, x0, x10
+; CHECK-NEXT:    fcmla z1.d, p0/m, z5.d, z3.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z4.d, z2.d, #0
+; CHECK-NEXT:    fcmla z1.d, p0/m, z5.d, z3.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z4.d, z2.d, #90
 ; CHECK-NEXT:    b.ne .LBB0_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
@@ -103,34 +100,31 @@ define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    fmov d0, #1.00000000
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    cntd x8
 ; CHECK-NEXT:    fmov d2, #2.00000000
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    neg x10, x9
-; CHECK-NEXT:    ptrue p1.b
-; CHECK-NEXT:    mov w11, #100 // =0x64
-; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    neg x9, x8
+; CHECK-NEXT:    mov w10, #100 // =0x64
 ; CHECK-NEXT:    sel z3.d, p0, z0.d, z1.d
-; CHECK-NEXT:    and x10, x10, x11
-; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    and x9, x9, x10
+; CHECK-NEXT:    rdvl x10, #2
 ; CHECK-NEXT:    mov z1.d, p0/m, z2.d
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z3.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z3.d
 ; CHECK-NEXT:  .LBB1_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x12, x0, x8
-; CHECK-NEXT:    add x13, x1, x8
-; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
-; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
-; CHECK-NEXT:    subs x10, x10, x9
-; CHECK-NEXT:    add x8, x8, x11
-; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #0
-; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #0
-; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #90
-; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #90
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT:    subs x9, x9, x8
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x1]
+; CHECK-NEXT:    add x1, x1, x10
+; CHECK-NEXT:    add x0, x0, x10
+; CHECK-NEXT:    fcmla z1.d, p0/m, z5.d, z3.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z4.d, z2.d, #0
+; CHECK-NEXT:    fcmla z1.d, p0/m, z5.d, z3.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z4.d, z2.d, #90
 ; CHECK-NEXT:    b.ne .LBB1_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
@@ -190,45 +184,37 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
 ; CHECK-LABEL: complex_mul_v2f64_unrolled:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov z1.d, #0 // =0x0
-; CHECK-NEXT:    cntw x9
-; CHECK-NEXT:    mov w11, #1000 // =0x3e8
-; CHECK-NEXT:    neg x10, x9
-; CHECK-NEXT:    rdvl x12, #2
-; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    cntw x8
+; CHECK-NEXT:    mov w10, #1000 // =0x3e8
+; CHECK-NEXT:    neg x9, x8
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    and x9, x9, x10
+; CHECK-NEXT:    rdvl x10, #4
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
-; CHECK-NEXT:    add x11, x1, x12
-; CHECK-NEXT:    add x12, x0, x12
-; CHECK-NEXT:    rdvl x13, #4
 ; CHECK-NEXT:    mov z2.d, z1.d
 ; CHECK-NEXT:    mov z3.d, z0.d
 ; CHECK-NEXT:  .LBB2_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x14, x0, x8
-; CHECK-NEXT:    add x15, x12, x8
-; CHECK-NEXT:    add x16, x1, x8
-; CHECK-NEXT:    add x17, x11, x8
-; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x14, #1, mul vl]
-; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x12, x8]
-; CHECK-NEXT:    ld1b { z7.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z16.d }, p0/z, [x16, #1, mul vl]
-; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x15, #1, mul vl]
-; CHECK-NEXT:    ld1b { z18.b }, p1/z, [x11, x8]
-; CHECK-NEXT:    ld1d { z19.d }, p0/z, [x17, #1, mul vl]
-; CHECK-NEXT:    subs x10, x10, x9
-; CHECK-NEXT:    add x8, x8, x13
-; CHECK-NEXT:    fcmla z1.d, p0/m, z7.d, z4.d, #0
-; CHECK-NEXT:    fcmla z0.d, p0/m, z16.d, z5.d, #0
-; CHECK-NEXT:    fcmla z2.d, p0/m, z18.d, z6.d, #0
-; CHECK-NEXT:    fcmla z3.d, p0/m, z19.d, z17.d, #0
-; CHECK-NEXT:    fcmla z1.d, p0/m, z7.d, z4.d, #90
-; CHECK-NEXT:    fcmla z0.d, p0/m, z16.d, z5.d, #90
-; CHECK-NEXT:    fcmla z2.d, p0/m, z18.d, z6.d, #90
-; CHECK-NEXT:    fcmla z3.d, p0/m, z19.d, z17.d, #90
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x0]
+; CHECK-NEXT:    subs x9, x9, x8
+; CHECK-NEXT:    ld1d { z6.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x1, #1, mul vl]
+; CHECK-NEXT:    ld1d { z16.d }, p0/z, [x1]
+; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    add x0, x0, x10
+; CHECK-NEXT:    ld1d { z18.d }, p0/z, [x1, #3, mul vl]
+; CHECK-NEXT:    ld1d { z19.d }, p0/z, [x1, #2, mul vl]
+; CHECK-NEXT:    add x1, x1, x10
+; CHECK-NEXT:    fcmla z1.d, p0/m, z16.d, z5.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #0
+; CHECK-NEXT:    fcmla z3.d, p0/m, z18.d, z6.d, #0
+; CHECK-NEXT:    fcmla z2.d, p0/m, z19.d, z17.d, #0
+; CHECK-NEXT:    fcmla z1.d, p0/m, z16.d, z5.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #90
+; CHECK-NEXT:    fcmla z3.d, p0/m, z18.d, z6.d, #90
+; CHECK-NEXT:    fcmla z2.d, p0/m, z19.d, z17.d, #90
 ; CHECK-NEXT:    b.ne .LBB2_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z4.d, z2.d, z3.d
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
index 44d0a9392ba629fb6fdf9154de67ad0e6fc475f8..aed3072bb4af377c365caf944dae1d23f13f288e 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
@@ -148,17 +148,16 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
 ; CHECK-NEXT:    adrp x8, .LCPI2_0
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    add x8, x0, #32
+; CHECK-NEXT:    add x9, x1, #32
+; CHECK-NEXT:    mov x10, #-100 // =0xffffffffffffff9c
 ; CHECK-NEXT:  .LBB2_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x0, x8
-; CHECK-NEXT:    add x10, x1, x8
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    ldp q5, q4, [x9]
-; CHECK-NEXT:    cmp x8, #1600
-; CHECK-NEXT:    ldp q7, q6, [x10]
-; CHECK-NEXT:    ldp q17, q16, [x9, #32]
-; CHECK-NEXT:    ldp q19, q18, [x10, #32]
+; CHECK-NEXT:    ldp q5, q4, [x8, #-32]
+; CHECK-NEXT:    adds x10, x10, #4
+; CHECK-NEXT:    ldp q7, q6, [x9, #-32]
+; CHECK-NEXT:    ldp q17, q16, [x8], #64
+; CHECK-NEXT:    ldp q19, q18, [x9], #64
 ; CHECK-NEXT:    fcmla v2.2d, v7.2d, v5.2d, #0
 ; CHECK-NEXT:    fcmla v0.2d, v6.2d, v4.2d, #0
 ; CHECK-NEXT:    fcmla v1.2d, v19.2d, v17.2d, #0
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index fa362ae798aa8ffcb1bd0747078635cfd8a4edad..998f7dd38c3f5042d397e18d80d8bf269ef64a44 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -759,6 +759,7 @@ declare half @llvm.sqrt.f16(half %a) #0
 declare half @llvm.powi.f16.i32(half %a, i32 %b) #0
 declare half @llvm.sin.f16(half %a) #0
 declare half @llvm.cos.f16(half %a) #0
+declare half @llvm.tan.f16(half %a) #0
 declare half @llvm.pow.f16(half %a, half %b) #0
 declare half @llvm.exp.f16(half %a) #0
 declare half @llvm.exp2.f16(half %a) #0
@@ -870,6 +871,31 @@ define half @test_cos(half %a) #0 {
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_tan
+; FALLBACK-FP16-NOT: remark:{{.*}}test_tan
+
+; CHECK-COMMON-LABEL: test_tan:
+; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
+; CHECK-COMMON-NEXT: mov  x29, sp
+; CHECK-COMMON-NEXT: fcvt s0, h0
+; CHECK-COMMON-NEXT: bl {{_?}}tanf
+; CHECK-COMMON-NEXT: fcvt h0, s0
+; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
+; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_tan:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}tanf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+define half @test_tan(half %a) #0 {
+  %r = call half @llvm.tan.f16(half %a)
+  ret half %r
+}
+
 ; CHECK-COMMON-LABEL: test_pow:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll b/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll
index 16bab1565e7fa6f83053fd53f9c13acb7fb234c6..7ff4dfca3f4c3056fe49a88926ec8caa1a78fd8b 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll
@@ -67,6 +67,28 @@ define double @cos_f64(double %a) {
   ret double %1
 }
 
+define float @tan_f32(float %a) {
+; SMALL-LABEL: tan_f32
+; SMALL:       bl _tanf
+; LARGE-LABEL: tan_f32
+; LARGE:       adrp  [[REG:x[0-9]+]], _tanf@GOTPAGE
+; LARGE:       ldr [[REG]], [[[REG]], _tanf@GOTPAGEOFF]
+; LARGE-NEXT:  blr [[REG]]
+  %1 = call float @llvm.tan.f32(float %a)
+  ret float %1
+}
+
+define double @tan_f64(double %a) {
+; SMALL-LABEL: tan_f64
+; SMALL:       bl _tan
+; LARGE-LABEL: tan_f64
+; LARGE:       adrp  [[REG:x[0-9]+]], _tan@GOTPAGE
+; LARGE:       ldr [[REG]], [[[REG]], _tan@GOTPAGEOFF]
+; LARGE-NEXT:  blr [[REG]]
+  %1 = call double @llvm.tan.f64(double %a)
+  ret double %1
+}
+
 define float @pow_f32(float %a, float %b) {
 ; SMALL-LABEL: pow_f32
 ; SMALL:       bl _powf
@@ -92,5 +114,7 @@ declare float @llvm.sin.f32(float)
 declare double @llvm.sin.f64(double)
 declare float @llvm.cos.f32(float)
 declare double @llvm.cos.f64(double)
+declare float @llvm.tan.f32(float)
+declare double @llvm.tan.f64(double)
 declare float @llvm.pow.f32(float, float)
 declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir b/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir
index f620cd22901ca9c21eaee4c500aa323e8d83624a..ff5ac24f713d0cfb08960d2a1c7025b9e2358cc4 100644
--- a/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir
+++ b/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple aarch64-apple-ios -run-pass regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple aarch64-apple-ios -passes=regallocfast -o - %s | FileCheck %s
 # This test used to crash the fast register alloc.
 # Basically, when a basic block has liveins, the fast regalloc
 # was deferencing the begin iterator of this block. However,
diff --git a/llvm/test/CodeGen/AArch64/fcvt-fixed.ll b/llvm/test/CodeGen/AArch64/fcvt-fixed.ll
index 296be831da7622cb6dd23763f7fa20c6617a3bb7..7056a4d28fed39845a7f3b5eebc50419fd087195 100644
--- a/llvm/test/CodeGen/AArch64/fcvt-fixed.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt-fixed.ll
@@ -412,10 +412,10 @@ define half @scvtf_f16_i32_7(i32 %int) {
 ; CHECK-NO16-LABEL: scvtf_f16_i32_7:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    scvtf s1, w0
-; CHECK-NO16-NEXT:    movi v0.2s, #67, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #60, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -432,10 +432,10 @@ define half @scvtf_f16_i32_15(i32 %int) {
 ; CHECK-NO16-LABEL: scvtf_f16_i32_15:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    scvtf s1, w0
-; CHECK-NO16-NEXT:    movi v0.2s, #71, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #56, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -452,10 +452,10 @@ define half @scvtf_f16_i64_7(i64 %long) {
 ; CHECK-NO16-LABEL: scvtf_f16_i64_7:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    scvtf s1, x0
-; CHECK-NO16-NEXT:    movi v0.2s, #67, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #60, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -472,10 +472,10 @@ define half @scvtf_f16_i64_15(i64 %long) {
 ; CHECK-NO16-LABEL: scvtf_f16_i64_15:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    scvtf s1, x0
-; CHECK-NO16-NEXT:    movi v0.2s, #71, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #56, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -574,10 +574,10 @@ define half @ucvtf_f16_i32_7(i32 %int) {
 ; CHECK-NO16-LABEL: ucvtf_f16_i32_7:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    ucvtf s1, w0
-; CHECK-NO16-NEXT:    movi v0.2s, #67, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #60, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -594,10 +594,10 @@ define half @ucvtf_f16_i32_15(i32 %int) {
 ; CHECK-NO16-LABEL: ucvtf_f16_i32_15:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    ucvtf s1, w0
-; CHECK-NO16-NEXT:    movi v0.2s, #71, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #56, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -614,10 +614,10 @@ define half @ucvtf_f16_i64_7(i64 %long) {
 ; CHECK-NO16-LABEL: ucvtf_f16_i64_7:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    ucvtf s1, x0
-; CHECK-NO16-NEXT:    movi v0.2s, #67, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #60, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -634,10 +634,10 @@ define half @ucvtf_f16_i64_15(i64 %long) {
 ; CHECK-NO16-LABEL: ucvtf_f16_i64_15:
 ; CHECK-NO16:       // %bb.0:
 ; CHECK-NO16-NEXT:    ucvtf s1, x0
-; CHECK-NO16-NEXT:    movi v0.2s, #71, lsl #24
+; CHECK-NO16-NEXT:    movi v0.2s, #56, lsl #24
 ; CHECK-NO16-NEXT:    fcvt h1, s1
 ; CHECK-NO16-NEXT:    fcvt s1, h1
-; CHECK-NO16-NEXT:    fdiv s0, s1, s0
+; CHECK-NO16-NEXT:    fmul s0, s1, s0
 ; CHECK-NO16-NEXT:    fcvt h0, s0
 ; CHECK-NO16-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/fdiv-const.ll b/llvm/test/CodeGen/AArch64/fdiv-const.ll
index 5a8f73384340114ee02e434559f48c18853c623a..7aa89db71adfebf9d8124d063b301a1ce0c132fe 100644
--- a/llvm/test/CodeGen/AArch64/fdiv-const.ll
+++ b/llvm/test/CodeGen/AArch64/fdiv-const.ll
@@ -4,8 +4,8 @@
 define float @divf32_2(float %a) nounwind {
 ; CHECK-LABEL: divf32_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov s1, #2.00000000
-; CHECK-NEXT:    fdiv s0, s0, s1
+; CHECK-NEXT:    fmov s1, #0.50000000
+; CHECK-NEXT:    fmul s0, s0, s1
 ; CHECK-NEXT:    ret
   %r = fdiv float %a, 2.0
   ret float %r
@@ -46,8 +46,8 @@ define float @divf32_p75_arcp(float %a) nounwind {
 define half @divf16_2(half %a) nounwind {
 ; CHECK-LABEL: divf16_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov h1, #2.00000000
-; CHECK-NEXT:    fdiv h0, h0, h1
+; CHECK-NEXT:    fmov h1, #0.50000000
+; CHECK-NEXT:    fmul h0, h0, h1
 ; CHECK-NEXT:    ret
   %r = fdiv half %a, 2.0
   ret half %r
@@ -67,9 +67,9 @@ define half @divf16_32768(half %a) nounwind {
 define half @divf16_32768_arcp(half %a) nounwind {
 ; CHECK-LABEL: divf16_32768_arcp:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #512 // =0x200
+; CHECK-NEXT:    mov w8, #30720 // =0x7800
 ; CHECK-NEXT:    fmov h1, w8
-; CHECK-NEXT:    fmul h0, h0, h1
+; CHECK-NEXT:    fdiv h0, h0, h1
 ; CHECK-NEXT:    ret
   %r = fdiv arcp half %a, 32768.0
   ret half %r
@@ -78,8 +78,8 @@ define half @divf16_32768_arcp(half %a) nounwind {
 define double @divf64_2(double %a) nounwind {
 ; CHECK-LABEL: divf64_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov d1, #2.00000000
-; CHECK-NEXT:    fdiv d0, d0, d1
+; CHECK-NEXT:    fmov d1, #0.50000000
+; CHECK-NEXT:    fmul d0, d0, d1
 ; CHECK-NEXT:    ret
   %r = fdiv double %a, 2.0
   ret double %r
@@ -88,8 +88,8 @@ define double @divf64_2(double %a) nounwind {
 define <4 x float> @divv4f32_2(<4 x float> %a) nounwind {
 ; CHECK-LABEL: divv4f32_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #64, lsl #24
-; CHECK-NEXT:    fdiv v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v1.4s, #63, lsl #24
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %r = fdiv <4 x float> %a, <float 2.0, float 2.0, float 2.0, float 2.0>
   ret <4 x float> %r
@@ -141,9 +141,8 @@ define <4 x float> @divv4f32_24816(<4 x float> %a) nounwind {
 define <vscale x 4 x float> @divnxv4f32_2(<vscale x 4 x float> %a) nounwind {
 ; CHECK-LABEL: divnxv4f32_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov z1.s, #2.00000000
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
   %r = fdiv <vscale x 4 x float> %a, splat (float 2.0)
   ret <vscale x 4 x float> %r
diff --git a/llvm/test/CodeGen/AArch64/frem-power2.ll b/llvm/test/CodeGen/AArch64/frem-power2.ll
index 4192745abd3477e589c4b4784f53b4041dcfc4ee..98276b68481a10ac0bc00fe6c252f033d38f8206 100644
--- a/llvm/test/CodeGen/AArch64/frem-power2.ll
+++ b/llvm/test/CodeGen/AArch64/frem-power2.ll
@@ -5,11 +5,12 @@
 define float @frem2(float %x) {
 ; CHECK-SD-LABEL: frem2:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    fmov s1, #0.50000000
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s1, s2, s1, s0
+; CHECK-SD-NEXT:    fmov s2, #-2.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s1, s1, s2, s0
 ; CHECK-SD-NEXT:    mvni v2.4s, #128, lsl #24
 ; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
@@ -27,10 +28,11 @@ entry:
 define float @frem2_nsz(float %x) {
 ; CHECK-SD-LABEL: frem2_nsz:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    fmov s1, #0.50000000
+; CHECK-SD-NEXT:    fmov s2, #-2.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_nsz:
@@ -65,10 +67,11 @@ define float @frem2_abs(float %x) {
 ; CHECK-SD-LABEL: frem2_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    fmov s1, #0.50000000
+; CHECK-SD-NEXT:    fmov s2, #-2.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_abs:
@@ -85,9 +88,9 @@ entry:
 define half @hrem2_nsz(half %x) {
 ; CHECK-SD-LABEL: hrem2_nsz:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov h1, #2.00000000
+; CHECK-SD-NEXT:    fmov h1, #0.50000000
 ; CHECK-SD-NEXT:    fmov h2, #-2.00000000
-; CHECK-SD-NEXT:    fdiv h1, h0, h1
+; CHECK-SD-NEXT:    fmul h1, h0, h1
 ; CHECK-SD-NEXT:    frintz h1, h1
 ; CHECK-SD-NEXT:    fmadd h0, h1, h2, h0
 ; CHECK-SD-NEXT:    ret
@@ -112,10 +115,11 @@ entry:
 define double @drem2_nsz(double %x) {
 ; CHECK-SD-LABEL: drem2_nsz:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov d1, #2.00000000
-; CHECK-SD-NEXT:    fdiv d2, d0, d1
-; CHECK-SD-NEXT:    frintz d2, d2
-; CHECK-SD-NEXT:    fmsub d0, d2, d1, d0
+; CHECK-SD-NEXT:    fmov d1, #0.50000000
+; CHECK-SD-NEXT:    fmov d2, #-2.00000000
+; CHECK-SD-NEXT:    fmul d1, d0, d1
+; CHECK-SD-NEXT:    frintz d1, d1
+; CHECK-SD-NEXT:    fmadd d0, d1, d2, d0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: drem2_nsz:
@@ -176,10 +180,11 @@ entry:
 define float @fremm2_nsz(float %x) {
 ; CHECK-SD-LABEL: fremm2_nsz:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s1, #-2.00000000
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    fmov s1, #-0.50000000
+; CHECK-SD-NEXT:    fmov s2, #2.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fremm2_nsz:
@@ -195,10 +200,11 @@ define float @frem4_abs(float %x) {
 ; CHECK-SD-LABEL: frem4_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    fmov s1, #4.00000000
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    fmov s1, #0.25000000
+; CHECK-SD-NEXT:    fmov s2, #-4.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem4_abs:
@@ -216,10 +222,12 @@ define float @frem16_abs(float %x) {
 ; CHECK-SD-LABEL: frem16_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    fmov s1, #16.00000000
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    mov w8, #1031798784 // =0x3d800000
+; CHECK-SD-NEXT:    fmov s2, #-16.00000000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem16_abs:
@@ -237,11 +245,13 @@ define float @frem4294967296_abs(float %x) {
 ; CHECK-SD-LABEL: frem4294967296_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    mov w8, #1333788672 // =0x4f800000
+; CHECK-SD-NEXT:    mov w8, #796917760 // =0x2f800000
 ; CHECK-SD-NEXT:    fmov s1, w8
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    mov w8, #-813694976 // =0xcf800000
+; CHECK-SD-NEXT:    fmov s2, w8
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem4294967296_abs:
@@ -260,11 +270,13 @@ define float @frem1152921504606846976_abs(float %x) {
 ; CHECK-SD-LABEL: frem1152921504606846976_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-SD-NEXT:    mov w8, #562036736 // =0x21800000
 ; CHECK-SD-NEXT:    fmov s1, w8
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    mov w8, #-578813952 // =0xdd800000
+; CHECK-SD-NEXT:    fmov s2, w8
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem1152921504606846976_abs:
@@ -283,11 +295,13 @@ define float @frem4611686018427387904_abs(float %x) {
 ; CHECK-SD-LABEL: frem4611686018427387904_abs:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    mov w8, #1585446912 // =0x5e800000
+; CHECK-SD-NEXT:    mov w8, #545259520 // =0x20800000
 ; CHECK-SD-NEXT:    fmov s1, w8
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    mov w8, #-562036736 // =0xde800000
+; CHECK-SD-NEXT:    fmov s2, w8
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem4611686018427387904_abs:
@@ -305,11 +319,12 @@ entry:
 define float @frem9223372036854775808_abs(float %x) {
 ; CHECK-SD-LABEL: frem9223372036854775808_abs:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2s, #95, lsl #24
+; CHECK-SD-NEXT:    movi v1.2s, #32, lsl #24
 ; CHECK-SD-NEXT:    fabs s0, s0
-; CHECK-SD-NEXT:    fdiv s2, s0, s1
-; CHECK-SD-NEXT:    frintz s2, s2
-; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    movi v2.2s, #223, lsl #24
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem9223372036854775808_abs:
@@ -326,11 +341,12 @@ entry:
 define <4 x float> @frem2_vec(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem2_vec:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.4s, #64, lsl #24
+; CHECK-SD-NEXT:    movi v1.4s, #63, lsl #24
+; CHECK-SD-NEXT:    movi v2.4s, #64, lsl #24
 ; CHECK-SD-NEXT:    mov v3.16b, v0.16b
-; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
-; CHECK-SD-NEXT:    fmls v3.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NEXT:    fmls v3.4s, v2.4s, v1.4s
 ; CHECK-SD-NEXT:    mvni v1.4s, #128, lsl #24
 ; CHECK-SD-NEXT:    bit v0.16b, v3.16b, v1.16b
 ; CHECK-SD-NEXT:    ret
@@ -387,10 +403,11 @@ entry:
 define <4 x float> @frem2_nsz_vec(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem2_nsz_vec:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.4s, #64, lsl #24
-; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
-; CHECK-SD-NEXT:    fmls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    movi v1.4s, #63, lsl #24
+; CHECK-SD-NEXT:    movi v2.4s, #64, lsl #24
+; CHECK-SD-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NEXT:    fmls v0.4s, v2.4s, v1.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_nsz_vec:
@@ -445,12 +462,14 @@ entry:
 define <4 x float> @frem1152921504606846976_absv(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem1152921504606846976_absv:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-SD-NEXT:    mov w8, #562036736 // =0x21800000
 ; CHECK-SD-NEXT:    fabs v0.4s, v0.4s
 ; CHECK-SD-NEXT:    dup v1.4s, w8
-; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
-; CHECK-SD-NEXT:    fmls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-SD-NEXT:    dup v2.4s, w8
+; CHECK-SD-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NEXT:    fmls v0.4s, v2.4s, v1.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem1152921504606846976_absv:
diff --git a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
index 20435e10d89ffe8688e000c8e5989404fb9fda35..3281a98767795fb24f4225463ff7c93318529212 100644
--- a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll
@@ -159,6 +159,28 @@ define void @test_sin(float %float, double %double, fp128 %fp128) {
 
 }
 
+declare float @llvm.tan.f32(float)
+declare double @llvm.tan.f64(double)
+declare fp128 @llvm.tan.f128(fp128)
+
+define void @test_tan(float %float, double %double, fp128 %fp128) {
+; CHECK-LABEL: test_tan:
+
+   %tanfloat = call float @llvm.tan.f32(float %float)
+   store float %tanfloat, ptr @varfloat
+; CHECK: bl tanf
+
+   %tandouble = call double @llvm.tan.f64(double %double)
+   store double %tandouble, ptr @vardouble
+; CHECK: bl tan
+
+   %tanfp128 = call fp128 @llvm.tan.f128(fp128 %fp128)
+   store fp128 %tanfp128, ptr @varfp128
+; CHECK: bl tanl
+  ret void
+
+}
+
 declare float @llvm.pow.f32(float, float)
 declare double @llvm.pow.f64(double, double)
 declare fp128 @llvm.pow.f128(fp128, fp128)
diff --git a/llvm/test/CodeGen/AArch64/insert-extend.ll b/llvm/test/CodeGen/AArch64/insert-extend.ll
index 0b730f6e771560a6ca38bf960940ed11f5c1c080..851fb0d03e8aa357002bfc598fb9b0a210820567 100644
--- a/llvm/test/CodeGen/AArch64/insert-extend.ll
+++ b/llvm/test/CodeGen/AArch64/insert-extend.ll
@@ -94,75 +94,73 @@ define i32 @large(ptr nocapture noundef readonly %p1, i32 noundef %st1, ptr noca
 ; CHECK-NEXT:    mov v3.d[1], v5.d[1]
 ; CHECK-NEXT:    uzp1 v1.4s, v4.4s, v0.4s
 ; CHECK-NEXT:    uzp2 v4.4s, v4.4s, v0.4s
-; CHECK-NEXT:    uzp2 v5.4s, v2.4s, v0.4s
-; CHECK-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    add v2.4s, v3.4s, v6.4s
+; CHECK-NEXT:    addp v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    add v5.4s, v3.4s, v6.4s
 ; CHECK-NEXT:    sub v3.4s, v6.4s, v3.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v5.4s, v0.4s
-; CHECK-NEXT:    rev64 v4.4s, v2.4s
-; CHECK-NEXT:    rev64 v5.4s, v3.4s
-; CHECK-NEXT:    rev64 v6.4s, v1.4s
 ; CHECK-NEXT:    rev64 v7.4s, v0.4s
-; CHECK-NEXT:    addp v16.4s, v1.4s, v3.4s
-; CHECK-NEXT:    addp v17.4s, v0.4s, v2.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v4.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    rev64 v4.4s, v5.4s
+; CHECK-NEXT:    rev64 v6.4s, v3.4s
+; CHECK-NEXT:    addp v16.4s, v0.4s, v5.4s
+; CHECK-NEXT:    rev64 v2.4s, v1.4s
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v7.4s
-; CHECK-NEXT:    zip1 v18.4s, v17.4s, v17.4s
-; CHECK-NEXT:    ext v4.16b, v17.16b, v2.16b, #4
-; CHECK-NEXT:    ext v5.16b, v16.16b, v3.16b, #4
+; CHECK-NEXT:    zip1 v21.4s, v16.4s, v16.4s
+; CHECK-NEXT:    sub v4.4s, v5.4s, v4.4s
+; CHECK-NEXT:    addp v5.4s, v1.4s, v3.4s
+; CHECK-NEXT:    sub v3.4s, v3.4s, v6.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    ext v7.16b, v0.16b, v16.16b, #4
+; CHECK-NEXT:    ext v2.16b, v16.16b, v4.16b, #4
+; CHECK-NEXT:    ext v6.16b, v5.16b, v3.16b, #4
+; CHECK-NEXT:    mov v19.16b, v4.16b
+; CHECK-NEXT:    ext v17.16b, v1.16b, v5.16b, #8
 ; CHECK-NEXT:    mov v20.16b, v3.16b
-; CHECK-NEXT:    ext v6.16b, v1.16b, v16.16b, #8
-; CHECK-NEXT:    ext v7.16b, v0.16b, v17.16b, #4
-; CHECK-NEXT:    mov v21.16b, v2.16b
-; CHECK-NEXT:    trn2 v0.4s, v18.4s, v0.4s
-; CHECK-NEXT:    mov v20.s[2], v16.s[3]
-; CHECK-NEXT:    zip2 v4.4s, v4.4s, v17.4s
-; CHECK-NEXT:    zip2 v5.4s, v5.4s, v16.4s
-; CHECK-NEXT:    mov v21.s[2], v17.s[3]
-; CHECK-NEXT:    ext v19.16b, v6.16b, v1.16b, #4
+; CHECK-NEXT:    trn2 v0.4s, v21.4s, v0.4s
 ; CHECK-NEXT:    ext v7.16b, v7.16b, v7.16b, #4
-; CHECK-NEXT:    mov v1.s[2], v16.s[1]
-; CHECK-NEXT:    ext v2.16b, v2.16b, v4.16b, #12
-; CHECK-NEXT:    ext v3.16b, v3.16b, v5.16b, #12
-; CHECK-NEXT:    uzp2 v4.4s, v6.4s, v19.4s
-; CHECK-NEXT:    mov v5.16b, v7.16b
-; CHECK-NEXT:    mov v6.16b, v20.16b
-; CHECK-NEXT:    mov v18.16b, v1.16b
-; CHECK-NEXT:    mov v19.16b, v21.16b
+; CHECK-NEXT:    mov v19.s[2], v16.s[3]
+; CHECK-NEXT:    zip2 v2.4s, v2.4s, v16.4s
+; CHECK-NEXT:    zip2 v6.4s, v6.4s, v5.4s
+; CHECK-NEXT:    mov v20.s[2], v5.s[3]
+; CHECK-NEXT:    ext v18.16b, v17.16b, v1.16b, #4
+; CHECK-NEXT:    mov v1.s[2], v5.s[1]
+; CHECK-NEXT:    mov v21.16b, v7.16b
 ; CHECK-NEXT:    sub v7.4s, v0.4s, v7.4s
-; CHECK-NEXT:    mov v6.s[1], v16.s[2]
-; CHECK-NEXT:    mov v5.s[0], v17.s[1]
-; CHECK-NEXT:    mov v18.s[1], v16.s[0]
-; CHECK-NEXT:    mov v19.s[1], v17.s[2]
+; CHECK-NEXT:    ext v2.16b, v4.16b, v2.16b, #12
+; CHECK-NEXT:    ext v3.16b, v3.16b, v6.16b, #12
+; CHECK-NEXT:    uzp2 v4.4s, v17.4s, v18.4s
+; CHECK-NEXT:    mov v6.16b, v1.16b
+; CHECK-NEXT:    mov v17.16b, v19.16b
+; CHECK-NEXT:    mov v18.16b, v20.16b
+; CHECK-NEXT:    mov v21.s[0], v16.s[1]
+; CHECK-NEXT:    mov v6.s[1], v5.s[0]
+; CHECK-NEXT:    mov v17.s[1], v16.s[2]
+; CHECK-NEXT:    sub v16.4s, v19.4s, v2.4s
+; CHECK-NEXT:    mov v18.s[1], v5.s[2]
 ; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    sub v16.4s, v20.4s, v3.4s
-; CHECK-NEXT:    sub v17.4s, v21.4s, v2.4s
-; CHECK-NEXT:    add v3.4s, v6.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v4.4s, v18.4s, v4.4s
-; CHECK-NEXT:    add v2.4s, v19.4s, v2.4s
-; CHECK-NEXT:    mov v3.d[1], v16.d[1]
+; CHECK-NEXT:    sub v5.4s, v20.4s, v3.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v21.4s
+; CHECK-NEXT:    add v4.4s, v6.4s, v4.4s
+; CHECK-NEXT:    add v2.4s, v17.4s, v2.4s
+; CHECK-NEXT:    add v3.4s, v18.4s, v3.4s
 ; CHECK-NEXT:    mov v0.d[1], v7.d[1]
 ; CHECK-NEXT:    mov v4.d[1], v1.d[1]
-; CHECK-NEXT:    mov v2.d[1], v17.d[1]
-; CHECK-NEXT:    cmlt v1.8h, v3.8h, #0
-; CHECK-NEXT:    cmlt v5.8h, v0.8h, #0
-; CHECK-NEXT:    cmlt v6.8h, v4.8h, #0
-; CHECK-NEXT:    cmlt v7.8h, v2.8h, #0
-; CHECK-NEXT:    add v3.4s, v1.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v5.4s, v0.4s
-; CHECK-NEXT:    add v4.4s, v6.4s, v4.4s
-; CHECK-NEXT:    add v2.4s, v7.4s, v2.4s
-; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    eor v0.16b, v0.16b, v5.16b
-; CHECK-NEXT:    eor v2.16b, v2.16b, v7.16b
-; CHECK-NEXT:    eor v3.16b, v4.16b, v6.16b
-; CHECK-NEXT:    add v0.4s, v0.4s, v3.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    mov v2.d[1], v16.d[1]
+; CHECK-NEXT:    mov v3.d[1], v5.d[1]
+; CHECK-NEXT:    cmlt v7.8h, v0.8h, #0
+; CHECK-NEXT:    cmlt v1.8h, v4.8h, #0
+; CHECK-NEXT:    cmlt v6.8h, v2.8h, #0
+; CHECK-NEXT:    cmlt v5.8h, v3.8h, #0
+; CHECK-NEXT:    add v0.4s, v7.4s, v0.4s
+; CHECK-NEXT:    add v4.4s, v1.4s, v4.4s
+; CHECK-NEXT:    add v2.4s, v6.4s, v2.4s
+; CHECK-NEXT:    add v3.4s, v5.4s, v3.4s
+; CHECK-NEXT:    eor v0.16b, v0.16b, v7.16b
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    eor v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    eor v3.16b, v3.16b, v5.16b
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    lsr w9, w8, #16
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll
new file mode 100644
index 0000000000000000000000000000000000000000..09b02531d2a7597c456b21389cf926647a01fb2d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll
@@ -0,0 +1,96 @@
+; This tests the order in which functions are outlined in MachineOutliner
+; There are TWO key OutlinedFunction in FunctionList
+;
+; ===================== First One =====================
+;   ```
+;     mov     w0, #1
+;     mov     w1, #2
+;     mov     w2, #3
+;     mov     w3, #4
+;     mov     w4, #5
+;   ```
+; It has:
+;   - `SequenceSize=20` and `OccurrenceCount=6`
+;   - each Candidate has `CallOverhead=12` and `FrameOverhead=4`
+;   - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96`
+;   - `Benefit=120-96=24` and `Priority=120/96=1.25`
+;
+; ===================== Second One =====================
+;   ```
+;     mov     w6, #6
+;     mov     w7, #7
+;     b
+;   ```
+; It has:
+;   - `SequenceSize=12` and `OccurrenceCount=4`
+;   - each Candidate has `CallOverhead=4` and `FrameOverhead=0`
+;   - `NotOutlinedCost=12*4=48` and `OutliningCost=4*4+12+0=28`
+;   - `Benefit=48-28=20` and `Priority=48/28=1.71`
+;
+; Note that the first one has higher benefit, but lower priority.
+; Hence, when outlining per priority, the second one will be outlined first.
+
+; RUN: llc %s -enable-machine-outliner=always -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-SORT-BY-PRIORITY
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-benefit-threshold=22 -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-THRESHOLD
+
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx14.0.0"
+
+declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+define i32 @_Z2f1v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 11, i32 noundef 6, i32 noundef 7)
+  ret i32 %1
+}
+
+define i32 @_Z2f2v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 12, i32 noundef 6, i32 noundef 7)
+  ret i32 %1
+}
+
+define i32 @_Z2f3v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 13, i32 noundef 6, i32 noundef 7)
+  ret i32 %1
+}
+
+define i32 @_Z2f4v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 14, i32 noundef 6, i32 noundef 7)
+  ret i32 %1
+}
+
+define i32 @_Z2f5v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 15, i32 noundef 8, i32 noundef 9)
+  ret i32 %1
+}
+
+define i32 @_Z2f6v() minsize {
+  %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 16, i32 noundef 9, i32 noundef 8)
+  ret i32 %1
+}
+
+; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_0>:
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w6, #0x6
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w7, #0x7
+; CHECK-SORT-BY-PRIORITY-NEXT: b
+
+; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_1>:
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w0, #0x1
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w1, #0x2
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w2, #0x3
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w3, #0x4
+; CHECK-SORT-BY-PRIORITY-NEXT: mov     w4, #0x5
+; CHECK-SORT-BY-PRIORITY-NEXT: ret
+
+; CHECK-THRESHOLD: <_OUTLINED_FUNCTION_0>:
+; CHECK-THRESHOLD-NEXT: mov     w0, #0x1
+; CHECK-THRESHOLD-NEXT: mov     w1, #0x2
+; CHECK-THRESHOLD-NEXT: mov     w2, #0x3
+; CHECK-THRESHOLD-NEXT: mov     w3, #0x4
+; CHECK-THRESHOLD-NEXT: mov     w4, #0x5
+; CHECK-THRESHOLD-NEXT: ret
+
+; CHECK-THRESHOLD-NOT: <_OUTLINED_FUNCTION_1>:
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir
new file mode 100644
index 0000000000000000000000000000000000000000..444590657a200a3b472ebcf33e009d54d33afc3b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir
@@ -0,0 +1,206 @@
+# The content of this test is modfied upon the output obtained from running
+# `bin/llc -O2 -stop-before=machine-outliner <path_to_llvm_project>/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll -o -`
+# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  declare i32 @foo()
+
+  define void @f1() #0 { ret void }
+  define void @f2() #0 { ret void }
+  define void @f3() #0 { ret void }
+  define void @f4() #0 { ret void }
+  define void @f5() #0 { ret void }
+  define void @f6() #0 { ret void }
+
+  attributes #0 = { minsize }
+...
+---
+# CHECK-LABEL: name:            f1
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NEXT: $w5 = MOVZWi 11, 0
+# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0
+name:            f1
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 11, 0
+    $w6 = MOVZWi 6, 0
+    $w7 = MOVZWi 7, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+---
+# CHECK-LABEL: name:            f2
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NEXT: $w5 = MOVZWi 12, 0
+# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0
+name:            f2
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 12, 0
+    $w6 = MOVZWi 6, 0
+    $w7 = MOVZWi 7, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+---
+# CHECK-LABEL: name:            f3
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NEXT: $w5 = MOVZWi 13, 0
+# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0
+name:            f3
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 13, 0
+    $w6 = MOVZWi 6, 0
+    $w7 = MOVZWi 7, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+---
+# CHECK-LABEL: name:            f4
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NEXT: $w5 = MOVZWi 14, 0
+# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0
+name:            f4
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 14, 0
+    $w6 = MOVZWi 6, 0
+    $w7 = MOVZWi 7, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+---
+# CHECK-LABEL: name:            f5
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NOT: @OUTLINED_FUNCTION_0
+name:            f5
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 15, 0
+    $w6 = MOVZWi 8, 0
+    $w7 = MOVZWi 9, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+---
+# CHECK-LABEL: name:            f6
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $lr
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0
+# CHECK-NEXT: BL @OUTLINED_FUNCTION_1
+# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0
+# CHECK-NOT: @OUTLINED_FUNCTION_0
+name:            f6
+tracksRegLiveness: true
+frameInfo:
+  isCalleeSavedInfoValid: true
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0:
+    $w0 = MOVZWi 1, 0
+    $w1 = MOVZWi 2, 0
+    $w2 = MOVZWi 3, 0
+    $w3 = MOVZWi 4, 0
+    $w4 = MOVZWi 5, 0
+    $w5 = MOVZWi 16, 0
+    $w6 = MOVZWi 9, 0
+    $w7 = MOVZWi 8, 0
+    TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7
+
+...
+
+# CHECK-LABEL: name:            OUTLINED_FUNCTION_0
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15, $w0, $w1, $w2, $w3, $w4, $w5
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $w6 = MOVZWi 6, 0
+# CHECK-NEXT: $w7 = MOVZWi 7, 0
+
+# CHECK-LABEL: name:            OUTLINED_FUNCTION_1
+# CHECK-LABEL: bb.0:
+# CHECK-NEXT: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15
+# CHECK-NEXT: {{ $}}
+# CHECK-NEXT: $w0 = MOVZWi 1, 0
+# CHECK-NEXT: $w1 = MOVZWi 2, 0
+# CHECK-NEXT: $w2 = MOVZWi 3, 0
+# CHECK-NEXT: $w3 = MOVZWi 4, 0
+# CHECK-NEXT: $w4 = MOVZWi 5, 0
+# CHECK-NEXT: RET $lr
diff --git a/llvm/test/CodeGen/AArch64/neon-extmul.ll b/llvm/test/CodeGen/AArch64/neon-extmul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..502673fa76694bffc8cd24dd262bba95ba8ae312
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-extmul.ll
@@ -0,0 +1,477 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple aarch64 -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple aarch64 -o - -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define <8 x i32> @extmuls_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extmuls_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v2.8h, v1.8b, #0
+; CHECK-SD-NEXT:    smull2 v1.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuls_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    smull v0.4s, v2.4h, v1.4h
+; CHECK-GI-NEXT:    smull2 v1.4s, v2.8h, v1.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i32>
+  %s1s = sext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  ret <8 x i32> %m
+}
+
+define <8 x i32> @extmulu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extmulu_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v2.8h, v1.8b, #0
+; CHECK-SD-NEXT:    umull2 v1.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    umull v0.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmulu_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    umull v0.4s, v2.4h, v1.4h
+; CHECK-GI-NEXT:    umull2 v1.4s, v2.8h, v1.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = zext <8 x i8> %s0 to <8 x i32>
+  %s1s = zext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  ret <8 x i32> %m
+}
+
+define <8 x i32> @extmulsu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extmulsu_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v2.8h, v1.8b, #0
+; CHECK-SD-NEXT:    smull2 v1.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmulsu_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    mul v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i32>
+  %s1s = zext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  ret <8 x i32> %m
+}
+
+define <8 x i32> @extmuladds_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
+; CHECK-SD-LABEL: extmuladds_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    smlal2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smlal v2.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladds_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    smlal v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    smlal2 v3.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i32>
+  %s1s = sext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  %a = add <8 x i32> %m, %b
+  ret <8 x i32> %a
+}
+
+define <8 x i32> @extmuladdu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
+; CHECK-SD-LABEL: extmuladdu_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    umlal2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    umlal v2.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladdu_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    umlal v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umlal2 v3.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = zext <8 x i8> %s0 to <8 x i32>
+  %s1s = zext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  %a = add <8 x i32> %m, %b
+  ret <8 x i32> %a
+}
+
+define <8 x i32> @extmuladdsu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
+; CHECK-SD-LABEL: extmuladdsu_v8i8_i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    smlal2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smlal v2.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladdsu_v8i8_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v4.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    mla v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mla v3.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i32>
+  %s1s = zext <8 x i8> %s1 to <8 x i32>
+  %m = mul <8 x i32> %s0s, %s1s
+  %a = add <8 x i32> %m, %b
+  ret <8 x i32> %a
+}
+
+
+
+define <8 x i64> @extmuls_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extmuls_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v4.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v5.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v6.4s, v1.8h, #0
+; CHECK-SD-NEXT:    smull v0.2d, v2.2s, v4.2s
+; CHECK-SD-NEXT:    smull2 v1.2d, v2.4s, v4.4s
+; CHECK-SD-NEXT:    smull2 v3.2d, v5.4s, v6.4s
+; CHECK-SD-NEXT:    smull v2.2d, v5.2s, v6.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuls_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll2 v5.4s, v1.8h, #0
+; CHECK-GI-NEXT:    smull v0.2d, v2.2s, v3.2s
+; CHECK-GI-NEXT:    smull2 v1.2d, v2.4s, v3.4s
+; CHECK-GI-NEXT:    smull v2.2d, v4.2s, v5.2s
+; CHECK-GI-NEXT:    smull2 v3.2d, v4.4s, v5.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i64>
+  %s1s = sext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  ret <8 x i64> %m
+}
+
+define <8 x i64> @extmulu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extmulu_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll2 v5.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v6.4s, v1.8h, #0
+; CHECK-SD-NEXT:    umull v0.2d, v2.2s, v4.2s
+; CHECK-SD-NEXT:    umull2 v1.2d, v2.4s, v4.4s
+; CHECK-SD-NEXT:    umull2 v3.2d, v5.4s, v6.4s
+; CHECK-SD-NEXT:    umull v2.2d, v5.2s, v6.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmulu_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll2 v5.4s, v1.8h, #0
+; CHECK-GI-NEXT:    umull v0.2d, v2.2s, v3.2s
+; CHECK-GI-NEXT:    umull2 v1.2d, v2.4s, v3.4s
+; CHECK-GI-NEXT:    umull v2.2d, v4.2s, v5.2s
+; CHECK-GI-NEXT:    umull2 v3.2d, v4.4s, v5.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = zext <8 x i8> %s0 to <8 x i64>
+  %s1s = zext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  ret <8 x i64> %m
+}
+
+define <8 x i64> @extaddsu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-SD-LABEL: extaddsu_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v5.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v6.4s, v1.8h, #0
+; CHECK-SD-NEXT:    smull v0.2d, v2.2s, v4.2s
+; CHECK-SD-NEXT:    smull2 v1.2d, v2.4s, v4.4s
+; CHECK-SD-NEXT:    smull2 v3.2d, v5.4s, v6.4s
+; CHECK-SD-NEXT:    smull v2.2d, v5.2s, v6.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extaddsu_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT:    fmov x8, d4
+; CHECK-GI-NEXT:    fmov x9, d5
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    fmov x11, d7
+; CHECK-GI-NEXT:    mov x13, v5.d[1]
+; CHECK-GI-NEXT:    fmov x14, d1
+; CHECK-GI-NEXT:    mov x15, v2.d[1]
+; CHECK-GI-NEXT:    mov x16, v3.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    mov x17, v7.d[1]
+; CHECK-GI-NEXT:    mov x18, v1.d[1]
+; CHECK-GI-NEXT:    mul x12, x12, x13
+; CHECK-GI-NEXT:    mov x13, v0.d[1]
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    fmov x10, d6
+; CHECK-GI-NEXT:    mul x15, x15, x16
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mul x13, x13, x18
+; CHECK-GI-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NEXT:    mul x11, x11, x14
+; CHECK-GI-NEXT:    mov x14, v6.d[1]
+; CHECK-GI-NEXT:    mov v1.d[1], x15
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    mul x14, x14, x17
+; CHECK-GI-NEXT:    fmov d3, x11
+; CHECK-GI-NEXT:    mov v3.d[1], x13
+; CHECK-GI-NEXT:    mov v2.d[1], x14
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i64>
+  %s1s = zext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  ret <8 x i64> %m
+}
+
+define <8 x i64> @extmuladds_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
+; CHECK-SD-LABEL: extmuladds_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v7.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-SD-NEXT:    smlal v2.2d, v6.2s, v7.2s
+; CHECK-SD-NEXT:    smlal2 v3.2d, v6.4s, v7.4s
+; CHECK-SD-NEXT:    smlal2 v5.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smlal v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    mov v2.16b, v4.16b
+; CHECK-SD-NEXT:    mov v3.16b, v5.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladds_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v7.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    smlal v2.2d, v6.2s, v7.2s
+; CHECK-GI-NEXT:    smlal2 v3.2d, v6.4s, v7.4s
+; CHECK-GI-NEXT:    smlal v4.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    smlal2 v5.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    mov v2.16b, v4.16b
+; CHECK-GI-NEXT:    mov v3.16b, v5.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i64>
+  %s1s = sext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  %a = add <8 x i64> %m, %b
+  ret <8 x i64> %a
+}
+
+define <8 x i64> @extmuladdu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
+; CHECK-SD-LABEL: extmuladdu_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v6.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-SD-NEXT:    umlal v2.2d, v6.2s, v7.2s
+; CHECK-SD-NEXT:    umlal2 v3.2d, v6.4s, v7.4s
+; CHECK-SD-NEXT:    umlal2 v5.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    umlal v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    mov v2.16b, v4.16b
+; CHECK-SD-NEXT:    mov v3.16b, v5.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladdu_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ushll v6.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    umlal v2.2d, v6.2s, v7.2s
+; CHECK-GI-NEXT:    umlal2 v3.2d, v6.4s, v7.4s
+; CHECK-GI-NEXT:    umlal v4.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    umlal2 v5.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    mov v2.16b, v4.16b
+; CHECK-GI-NEXT:    mov v3.16b, v5.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = zext <8 x i8> %s0 to <8 x i64>
+  %s1s = zext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  %a = add <8 x i64> %m, %b
+  ret <8 x i64> %a
+}
+
+define <8 x i64> @extmuladdsu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
+; CHECK-SD-LABEL: extmuladdsu_v8i8_i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-SD-NEXT:    smlal v2.2d, v6.2s, v7.2s
+; CHECK-SD-NEXT:    smlal2 v3.2d, v6.4s, v7.4s
+; CHECK-SD-NEXT:    smlal2 v5.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smlal v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NEXT:    mov v2.16b, v4.16b
+; CHECK-SD-NEXT:    mov v3.16b, v5.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: extmuladdsu_v8i8_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    sshll v16.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v17.2d, v7.2s, #0
+; CHECK-GI-NEXT:    sshll2 v6.2d, v6.4s, #0
+; CHECK-GI-NEXT:    ushll2 v7.2d, v7.4s, #0
+; CHECK-GI-NEXT:    sshll v18.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v19.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT:    fmov x8, d16
+; CHECK-GI-NEXT:    fmov x9, d17
+; CHECK-GI-NEXT:    mov x12, v16.d[1]
+; CHECK-GI-NEXT:    fmov x10, d7
+; CHECK-GI-NEXT:    fmov x11, d19
+; CHECK-GI-NEXT:    mov x13, v17.d[1]
+; CHECK-GI-NEXT:    fmov x14, d1
+; CHECK-GI-NEXT:    mov x15, v6.d[1]
+; CHECK-GI-NEXT:    mov x16, v7.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d6
+; CHECK-GI-NEXT:    mov x17, v19.d[1]
+; CHECK-GI-NEXT:    mov x18, v1.d[1]
+; CHECK-GI-NEXT:    mul x12, x12, x13
+; CHECK-GI-NEXT:    mov x13, v0.d[1]
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    fmov x10, d18
+; CHECK-GI-NEXT:    mul x15, x15, x16
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mul x13, x13, x18
+; CHECK-GI-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NEXT:    mul x11, x11, x14
+; CHECK-GI-NEXT:    mov x14, v18.d[1]
+; CHECK-GI-NEXT:    mov v1.d[1], x15
+; CHECK-GI-NEXT:    fmov d6, x10
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    mul x14, x14, x17
+; CHECK-GI-NEXT:    add v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    fmov d7, x11
+; CHECK-GI-NEXT:    mov v7.d[1], x13
+; CHECK-GI-NEXT:    mov v6.d[1], x14
+; CHECK-GI-NEXT:    add v3.2d, v7.2d, v5.2d
+; CHECK-GI-NEXT:    add v2.2d, v6.2d, v4.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s0s = sext <8 x i8> %s0 to <8 x i64>
+  %s1s = zext <8 x i8> %s1 to <8 x i64>
+  %m = mul <8 x i64> %s0s, %s1s
+  %a = add <8 x i64> %m, %b
+  ret <8 x i64> %a
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
index 758df0493cc50465ae7a8d83ee1fe7f7e4622c69..7d23e870637a8ace38d8595fa046c6b5bf95f1a7 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
@@ -15,7 +15,7 @@ declare <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float>)
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [36 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vfmodq_f64, ptr @armpl_vfmodq_f32, ptr @armpl_svfmod_f64_x, ptr @armpl_svfmod_f32_x], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [36 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x], section "llvm.metadata"
 ;.
 define <2 x double> @llvm_cos_f64(<2 x double> %in) {
 ; CHECK-LABEL: define <2 x double> @llvm_cos_f64
@@ -424,44 +424,48 @@ define <vscale x 4 x float> @llvm_sin_vscale_f32(<vscale x 4 x float> %in) #0 {
   ret <vscale x 4 x float> %1
 }
 
+declare <2 x double> @llvm.tan.v2f64(<2 x double>)
+declare <4 x float> @llvm.tan.v4f32(<4 x float>)
+declare <vscale x 2 x double> @llvm.tan.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.tan.nxv4f32(<vscale x 4 x float>)
 
-define <2 x double> @frem_f64(<2 x double> %in) {
-; CHECK-LABEL: define <2 x double> @frem_f64
+define <2 x double> @llvm_tan_f64(<2 x double> %in) {
+; CHECK-LABEL: define <2 x double> @llvm_tan_f64
 ; CHECK-SAME: (<2 x double> [[IN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @armpl_vfmodq_f64(<2 x double> [[IN]], <2 x double> [[IN]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vtanq_f64(<2 x double> [[IN]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
-  %1= frem <2 x double> %in, %in
+  %1 = call fast <2 x double> @llvm.tan.v2f64(<2 x double> %in)
   ret <2 x double> %1
 }
 
-define <4 x float> @frem_f32(<4 x float> %in) {
-; CHECK-LABEL: define <4 x float> @frem_f32
+define <4 x float> @llvm_tan_f32(<4 x float> %in) {
+; CHECK-LABEL: define <4 x float> @llvm_tan_f32
 ; CHECK-SAME: (<4 x float> [[IN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @armpl_vfmodq_f32(<4 x float> [[IN]], <4 x float> [[IN]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vtanq_f32(<4 x float> [[IN]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
-  %1= frem <4 x float> %in, %in
+  %1 = call fast <4 x float> @llvm.tan.v4f32(<4 x float> %in)
   ret <4 x float> %1
 }
 
-define <vscale x 2 x double> @frem_vscale_f64(<vscale x 2 x double> %in) #0 {
-; CHECK-LABEL: define <vscale x 2 x double> @frem_vscale_f64
+define <vscale x 2 x double> @llvm_tan_vscale_f64(<vscale x 2 x double> %in) #0 {
+; CHECK-LABEL: define <vscale x 2 x double> @llvm_tan_vscale_f64
 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 ;
-  %1= frem <vscale x 2 x double> %in, %in
+  %1 = call fast <vscale x 2 x double> @llvm.tan.nxv2f64(<vscale x 2 x double> %in)
   ret <vscale x 2 x double> %1
 }
 
-define <vscale x 4 x float> @frem_vscale_f32(<vscale x 4 x float> %in) #0 {
-; CHECK-LABEL: define <vscale x 4 x float> @frem_vscale_f32
+define <vscale x 4 x float> @llvm_tan_vscale_f32(<vscale x 4 x float> %in) #0 {
+; CHECK-LABEL: define <vscale x 4 x float> @llvm_tan_vscale_f32
 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 ;
-  %1= frem <vscale x 4 x float> %in, %in
+  %1 = call fast <vscale x 4 x float> @llvm.tan.nxv4f32(<vscale x 4 x float> %in)
   ret <vscale x 4 x float> %1
 }
 
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
index 9e214655e413a01c2f1c0837c6bb9d9767c0f162..15d100a518c1509ad02e8ae174091d09026d11e8 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxvv_fmod, ptr @_ZGVsMxvv_fmodf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf], section "llvm.metadata"
 ;.
 define <vscale x 2 x double> @llvm_ceil_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_vscale_f64(
@@ -366,39 +366,40 @@ define <vscale x 4 x float> @llvm_sqrt_vscale_f32(<vscale x 4 x float> %in) {
   ret <vscale x 4 x float> %1
 }
 
-define <vscale x 2 x double> @llvm_trunc_vscale_f64(<vscale x 2 x double> %in) {
-; CHECK-LABEL: @llvm_trunc_vscale_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[IN:%.*]])
+define <vscale x 2 x double> @llvm_tan_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_tan_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 ;
-  %1 = call fast <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> %in)
+  %1 = call fast <vscale x 2 x double> @llvm.tan.nxv2f64(<vscale x 2 x double> %in)
   ret <vscale x 2 x double> %1
 }
 
-define <vscale x 4 x float> @llvm_trunc_vscale_f32(<vscale x 4 x float> %in) {
-; CHECK-LABEL: @llvm_trunc_vscale_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[IN:%.*]])
+define <vscale x 4 x float> @llvm_tan_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_tan_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 ;
-  %1 = call fast <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> %in)
+  %1 = call fast <vscale x 4 x float> @llvm.tan.nxv4f32(<vscale x 4 x float> %in)
   ret <vscale x 4 x float> %1
 }
 
-define <vscale x 2 x double> @frem_f64(<vscale x 2 x double> %in) {
-; CHECK-LABEL: @frem_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+
+define <vscale x 2 x double> @llvm_trunc_vscale_f64(<vscale x 2 x double> %in) {
+; CHECK-LABEL: @llvm_trunc_vscale_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[IN:%.*]])
 ; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 ;
-  %1= frem <vscale x 2 x double> %in, %in
+  %1 = call fast <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> %in)
   ret <vscale x 2 x double> %1
 }
 
-define <vscale x 4 x float> @frem_f32(<vscale x 4 x float> %in) {
-; CHECK-LABEL: @frem_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+define <vscale x 4 x float> @llvm_trunc_vscale_f32(<vscale x 4 x float> %in) {
+; CHECK-LABEL: @llvm_trunc_vscale_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[IN:%.*]])
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 ;
-  %1= frem <vscale x 4 x float> %in, %in
+  %1 = call fast <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> %in)
   ret <vscale x 4 x float> %1
 }
 
@@ -442,6 +443,8 @@ declare <vscale x 2 x double> @llvm.sin.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.tan.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.tan.nxv4f32(<vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float>)
 ;.
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
index f408df570fdc00da01ac24f3e8d55e1cff4cbf8a..a3da3b8120218c9aed75faa4553dc95f778fd6de 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2vv_fmod, ptr @_ZGVnN4vv_fmodf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf], section "llvm.metadata"
 ;.
 define <2 x double> @llvm_ceil_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_f64(
@@ -366,39 +366,39 @@ define <4 x float> @llvm_sqrt_f32(<4 x float> %in) {
   ret <4 x float> %1
 }
 
-define <2 x double> @llvm_trunc_f64(<2 x double> %in) {
-; CHECK-LABEL: @llvm_trunc_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> [[IN:%.*]])
+define <2 x double> @llvm_tan_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_tan_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_tan(<2 x double> [[IN:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
-  %1 = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> %in)
+  %1 = call fast <2 x double> @llvm.tan.v2f64(<2 x double> %in)
   ret <2 x double> %1
 }
 
-define <4 x float> @llvm_trunc_f32(<4 x float> %in) {
-; CHECK-LABEL: @llvm_trunc_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> [[IN:%.*]])
+define <4 x float> @llvm_tan_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_tan_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_tanf(<4 x float> [[IN:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
-  %1 = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> %in)
+  %1 = call fast <4 x float> @llvm.tan.v4f32(<4 x float> %in)
   ret <4 x float> %1
 }
 
-define <2 x double> @frem_f64(<2 x double> %in) {
-; CHECK-LABEL: @frem_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @_ZGVnN2vv_fmod(<2 x double> [[IN:%.*]], <2 x double> [[IN]])
+define <2 x double> @llvm_trunc_f64(<2 x double> %in) {
+; CHECK-LABEL: @llvm_trunc_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> [[IN:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
-  %1= frem <2 x double> %in, %in
+  %1 = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> %in)
   ret <2 x double> %1
 }
 
-define <4 x float> @frem_f32(<4 x float> %in) {
-; CHECK-LABEL: @frem_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @_ZGVnN4vv_fmodf(<4 x float> [[IN:%.*]], <4 x float> [[IN]])
+define <4 x float> @llvm_trunc_f32(<4 x float> %in) {
+; CHECK-LABEL: @llvm_trunc_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> [[IN:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
-  %1= frem <4 x float> %in, %in
+  %1 = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> %in)
   ret <4 x float> %1
 }
 
@@ -442,6 +442,8 @@ declare <2 x double> @llvm.sin.v2f64(<2 x double>)
 declare <4 x float> @llvm.sin.v4f32(<4 x float>)
 declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
+declare <2 x double> @llvm.tan.v2f64(<2 x double>)
+declare <4 x float> @llvm.tan.v4f32(<4 x float>)
 declare <2 x double> @llvm.trunc.v2f64(<2 x double>)
 declare <4 x float> @llvm.trunc.v4f32(<4 x float>)
 ;.
diff --git a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
index 7535638137caa84e982d37b7688619fe5875a04e..63c65334afe1195ee5143069e1e53e8aefbf781d 100644
--- a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
+++ b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
@@ -25,6 +25,25 @@ define void @test_sme_calling_convention_x0() nounwind {
   ret void
 }
 
+define i64 @test_sme_calling_convention_x1() nounwind {
+; CHECK-LABEL: test_sme_calling_convention_x1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl __arm_get_current_vg
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+; DARWIN-LABEL: test_sme_calling_convention_x1:
+; DARWIN:    stp        x29, x30, [sp, #-16]!
+; DARWIN:    bl ___arm_get_current_vg
+; DARWIN:    ldp        x29, x30, [sp], #16
+; DARWIN:    ret
+;
+; CHECK-CSRMASK-LABEL: name: test_sme_calling_convention_x1
+; CHECK-CSRMASK: BL @__arm_get_current_vg, csr_aarch64_sme_abi_support_routines_preservemost_from_x1
+  %vg = call aarch64_sme_preservemost_from_x1 i64 @__arm_get_current_vg()
+  ret i64 %vg
+}
+
 define i64 @test_sme_calling_convention_x2() nounwind {
 ; CHECK-LABEL: test_sme_calling_convention_x2:
 ; CHECK:       // %bb.0:
@@ -46,4 +65,5 @@ define i64 @test_sme_calling_convention_x2() nounwind {
 }
 
 declare void @__arm_tpidr2_save()
+declare i64 @__arm_get_current_vg()
 declare {i64, i64} @__arm_sme_state()
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll
index e07645c27df728d004dfd94f8fc2cc02f2ccc5e8..fba324cfd74ac5a04ca8dae1936ee3e04c635e0c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll
@@ -611,14 +611,14 @@ define void @uzp_v8i16(ptr %a, ptr %b) #1 {
 ; CHECK-NEXT:    ldr q1, [x1]
 ; CHECK-NEXT:    uzp1 v2.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    add v0.8h, v2.8h, v0.8h
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, ptr %a
   %tmp2 = load <8 x i16>, ptr %b
   %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %tmp4 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
+  %tmp5 = xor <8 x i16> %tmp3, %tmp4
   store <8 x i16> %tmp5, ptr %a
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
index aa2cca40a92c2cf721d7ff4895b35500f9d5dba5..9bbac5d69c28c084773f244023a503c7ef22019d 100644
--- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll
+++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
@@ -20,6 +20,7 @@ declare <3 x double> @llvm.sin.v3f64(<3 x double>)
 declare <3 x float> @llvm.fabs.v3f32(<3 x float>)
 declare <3 x float> @llvm.ceil.v3f32(<3 x float>)
 declare <3 x float> @llvm.cos.v3f32(<3 x float>)
+declare <3 x float> @llvm.tan.v3f32(<3 x float>)
 declare <3 x float> @llvm.exp.v3f32(<3 x float>)
 declare <3 x float> @llvm.exp2.v3f32(<3 x float>)
 declare <3 x float> @llvm.floor.v3f32(<3 x float>)
@@ -297,6 +298,37 @@ define <3 x float> @cos_v3f32(<3 x float> %x) nounwind {
   ret <3 x float> %r
 }
 
+define <3 x float> @tan_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: tan_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl tanf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl tanf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl tanf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.tan.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
 define <3 x float> @exp_v3f32(<3 x float> %x) nounwind {
 ; CHECK-LABEL: exp_v3f32:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
index 4c02a5240ba6affea70206533e79e6db28c97060..c993051ccebf7cb45aff018ff05cb571823782fc 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
@@ -648,7 +648,19 @@ define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
 ; CHECK-LABEL: test_v2f128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    b fmaxl
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
+; CHECK-NEXT:    bl __gttf2
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    cmp w0, #0
+; CHECK-NEXT:    b.le .LBB18_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:  .LBB18_2:
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
   %b = call nnan fp128 @llvm.vector.reduce.fmax.v2f128(<2 x fp128> %a)
   ret fp128 %b
 }
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
index 18d40cb18ba60982fed39c49d22f98e904b2b7ca..0116be51dd696a9f3d937337945b7dba7f0f4770 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
@@ -648,7 +648,19 @@ define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
 ; CHECK-LABEL: test_v2f128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    b fminl
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
+; CHECK-NEXT:    bl __lttf2
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    cmp w0, #0
+; CHECK-NEXT:    b.ge .LBB18_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:  .LBB18_2:
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
   %b = call nnan fp128 @llvm.vector.reduce.fmin.v2f128(<2 x fp128> %a)
   ret fp128 %b
 }
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 599bd811d7d5988fe6136ffc510771c58bbdac45..66bb131ce724949a40511d7538b73b3529c9d64e 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1669,42 +1669,41 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst)
 ; CHECK-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:  Lloh18:
-; CHECK-NEXT:    adrp x9, lCPI17_0@PAGE
+; CHECK-NEXT:    adrp x8, lCPI17_0@PAGE
 ; CHECK-NEXT:  Lloh19:
-; CHECK-NEXT:    adrp x10, lCPI17_1@PAGE
-; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    adrp x9, lCPI17_1@PAGE
+; CHECK-NEXT:    mov w10, #128 ; =0x80
 ; CHECK-NEXT:  Lloh20:
-; CHECK-NEXT:    ldr q0, [x9, lCPI17_0@PAGEOFF]
+; CHECK-NEXT:    ldr q0, [x8, lCPI17_0@PAGEOFF]
 ; CHECK-NEXT:  Lloh21:
-; CHECK-NEXT:    ldr q1, [x10, lCPI17_1@PAGEOFF]
+; CHECK-NEXT:    ldr q1, [x9, lCPI17_1@PAGEOFF]
+; CHECK-NEXT:    add x8, x1, #64
 ; CHECK-NEXT:    add x9, x0, #8
 ; CHECK-NEXT:  LBB17_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldp d2, d3, [x9, #-8]
-; CHECK-NEXT:    add x10, x1, x8
-; CHECK-NEXT:    ldp q6, q5, [x10, #32]
-; CHECK-NEXT:    add x8, x8, #128
-; CHECK-NEXT:    ldp q17, q16, [x10]
-; CHECK-NEXT:    cmp x8, #1024
+; CHECK-NEXT:    subs x10, x10, #16
+; CHECK-NEXT:    ldp q6, q5, [x8, #-32]
+; CHECK-NEXT:    add x9, x9, #16
+; CHECK-NEXT:    ldp q17, q16, [x8, #-64]
 ; CHECK-NEXT:    tbl.16b v4, { v2 }, v1
 ; CHECK-NEXT:    tbl.16b v2, { v2 }, v0
 ; CHECK-NEXT:    tbl.16b v7, { v3 }, v1
 ; CHECK-NEXT:    tbl.16b v3, { v3 }, v0
-; CHECK-NEXT:    add x9, x9, #16
 ; CHECK-NEXT:    uaddw2.2d v5, v5, v4
 ; CHECK-NEXT:    uaddw.2d v4, v6, v4
 ; CHECK-NEXT:    uaddw2.2d v6, v16, v2
-; CHECK-NEXT:    ldp q18, q16, [x10, #96]
+; CHECK-NEXT:    ldp q18, q16, [x8, #32]
 ; CHECK-NEXT:    uaddw.2d v2, v17, v2
-; CHECK-NEXT:    stp q4, q5, [x10, #32]
+; CHECK-NEXT:    stp q4, q5, [x8, #-32]
 ; CHECK-NEXT:    uaddw2.2d v5, v16, v7
-; CHECK-NEXT:    ldp q16, q4, [x10, #64]
+; CHECK-NEXT:    ldp q16, q4, [x8]
 ; CHECK-NEXT:    uaddw.2d v7, v18, v7
-; CHECK-NEXT:    stp q2, q6, [x10]
+; CHECK-NEXT:    stp q2, q6, [x8, #-64]
 ; CHECK-NEXT:    uaddw2.2d v4, v4, v3
 ; CHECK-NEXT:    uaddw.2d v2, v16, v3
-; CHECK-NEXT:    stp q7, q5, [x10, #96]
-; CHECK-NEXT:    stp q2, q4, [x10, #64]
+; CHECK-NEXT:    stp q7, q5, [x8, #32]
+; CHECK-NEXT:    stp q2, q4, [x8], #128
 ; CHECK-NEXT:    b.ne LBB17_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
@@ -1715,67 +1714,67 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst)
 ; CHECK-BE:       // %bb.0: // %entry
 ; CHECK-BE-NEXT:    adrp x9, .LCPI17_0
 ; CHECK-BE-NEXT:    add x9, x9, :lo12:.LCPI17_0
-; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:    mov w8, #128 // =0x80
 ; CHECK-BE-NEXT:    ld1 { v0.16b }, [x9]
 ; CHECK-BE-NEXT:    adrp x9, .LCPI17_1
 ; CHECK-BE-NEXT:    add x9, x9, :lo12:.LCPI17_1
 ; CHECK-BE-NEXT:    ld1 { v1.16b }, [x9]
-; CHECK-BE-NEXT:    add x9, x0, #8
+; CHECK-BE-NEXT:    add x9, x1, #64
+; CHECK-BE-NEXT:    add x10, x0, #8
 ; CHECK-BE-NEXT:  .LBB17_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    sub x10, x9, #8
-; CHECK-BE-NEXT:    ld1 { v2.8b }, [x9]
-; CHECK-BE-NEXT:    add x9, x9, #16
-; CHECK-BE-NEXT:    ld1 { v3.8b }, [x10]
-; CHECK-BE-NEXT:    add x10, x1, x8
-; CHECK-BE-NEXT:    add x8, x8, #128
-; CHECK-BE-NEXT:    add x11, x10, #32
-; CHECK-BE-NEXT:    add x14, x10, #64
-; CHECK-BE-NEXT:    add x15, x10, #96
+; CHECK-BE-NEXT:    ld1 { v2.8b }, [x10]
+; CHECK-BE-NEXT:    sub x11, x10, #8
+; CHECK-BE-NEXT:    add x15, x9, #32
+; CHECK-BE-NEXT:    ld1 { v3.8b }, [x11]
+; CHECK-BE-NEXT:    ld1 { v16.2d }, [x15]
+; CHECK-BE-NEXT:    sub x11, x9, #64
+; CHECK-BE-NEXT:    sub x12, x9, #32
+; CHECK-BE-NEXT:    ld1 { v6.2d }, [x9]
+; CHECK-BE-NEXT:    ld1 { v21.2d }, [x11]
 ; CHECK-BE-NEXT:    tbl v4.16b, { v2.16b }, v1.16b
 ; CHECK-BE-NEXT:    tbl v2.16b, { v2.16b }, v0.16b
-; CHECK-BE-NEXT:    ld1 { v5.2d }, [x10]
-; CHECK-BE-NEXT:    tbl v6.16b, { v3.16b }, v1.16b
+; CHECK-BE-NEXT:    ld1 { v19.2d }, [x12]
+; CHECK-BE-NEXT:    tbl v5.16b, { v3.16b }, v1.16b
 ; CHECK-BE-NEXT:    tbl v3.16b, { v3.16b }, v0.16b
-; CHECK-BE-NEXT:    ld1 { v16.2d }, [x15]
-; CHECK-BE-NEXT:    ld1 { v19.2d }, [x14]
-; CHECK-BE-NEXT:    ld1 { v21.2d }, [x11]
-; CHECK-BE-NEXT:    add x12, x10, #48
-; CHECK-BE-NEXT:    add x13, x10, #16
-; CHECK-BE-NEXT:    add x16, x10, #112
-; CHECK-BE-NEXT:    add x17, x10, #80
+; CHECK-BE-NEXT:    sub x13, x9, #16
+; CHECK-BE-NEXT:    sub x14, x9, #48
+; CHECK-BE-NEXT:    add x16, x9, #48
+; CHECK-BE-NEXT:    add x17, x9, #16
+; CHECK-BE-NEXT:    ld1 { v22.2d }, [x13]
+; CHECK-BE-NEXT:    subs x8, x8, #16
+; CHECK-BE-NEXT:    add x10, x10, #16
 ; CHECK-BE-NEXT:    rev32 v7.8b, v4.8b
 ; CHECK-BE-NEXT:    ext v4.16b, v4.16b, v4.16b, #8
 ; CHECK-BE-NEXT:    rev32 v17.8b, v2.8b
-; CHECK-BE-NEXT:    ext v18.16b, v6.16b, v6.16b, #8
+; CHECK-BE-NEXT:    ext v18.16b, v5.16b, v5.16b, #8
 ; CHECK-BE-NEXT:    ext v20.16b, v3.16b, v3.16b, #8
 ; CHECK-BE-NEXT:    ext v2.16b, v2.16b, v2.16b, #8
-; CHECK-BE-NEXT:    rev32 v6.8b, v6.8b
+; CHECK-BE-NEXT:    rev32 v5.8b, v5.8b
 ; CHECK-BE-NEXT:    rev32 v3.8b, v3.8b
-; CHECK-BE-NEXT:    ld1 { v22.2d }, [x12]
-; CHECK-BE-NEXT:    cmp x8, #1024
-; CHECK-BE-NEXT:    rev32 v4.8b, v4.8b
 ; CHECK-BE-NEXT:    uaddw v7.2d, v16.2d, v7.2s
-; CHECK-BE-NEXT:    ld1 { v16.2d }, [x16]
-; CHECK-BE-NEXT:    rev32 v18.8b, v18.8b
+; CHECK-BE-NEXT:    rev32 v4.8b, v4.8b
+; CHECK-BE-NEXT:    uaddw v6.2d, v6.2d, v17.2s
+; CHECK-BE-NEXT:    rev32 v17.8b, v18.8b
 ; CHECK-BE-NEXT:    rev32 v20.8b, v20.8b
 ; CHECK-BE-NEXT:    rev32 v2.8b, v2.8b
-; CHECK-BE-NEXT:    uaddw v17.2d, v19.2d, v17.2s
-; CHECK-BE-NEXT:    ld1 { v19.2d }, [x13]
-; CHECK-BE-NEXT:    uaddw v6.2d, v21.2d, v6.2s
-; CHECK-BE-NEXT:    uaddw v3.2d, v5.2d, v3.2s
-; CHECK-BE-NEXT:    ld1 { v5.2d }, [x17]
+; CHECK-BE-NEXT:    ld1 { v16.2d }, [x16]
+; CHECK-BE-NEXT:    ld1 { v18.2d }, [x14]
+; CHECK-BE-NEXT:    uaddw v5.2d, v19.2d, v5.2s
+; CHECK-BE-NEXT:    uaddw v3.2d, v21.2d, v3.2s
 ; CHECK-BE-NEXT:    st1 { v7.2d }, [x15]
+; CHECK-BE-NEXT:    ld1 { v7.2d }, [x17]
+; CHECK-BE-NEXT:    st1 { v6.2d }, [x9]
+; CHECK-BE-NEXT:    add x9, x9, #128
 ; CHECK-BE-NEXT:    uaddw v4.2d, v16.2d, v4.2s
-; CHECK-BE-NEXT:    st1 { v6.2d }, [x11]
-; CHECK-BE-NEXT:    uaddw v6.2d, v22.2d, v18.2s
-; CHECK-BE-NEXT:    st1 { v3.2d }, [x10]
-; CHECK-BE-NEXT:    uaddw v3.2d, v19.2d, v20.2s
-; CHECK-BE-NEXT:    uaddw v2.2d, v5.2d, v2.2s
-; CHECK-BE-NEXT:    st1 { v17.2d }, [x14]
+; CHECK-BE-NEXT:    st1 { v5.2d }, [x12]
+; CHECK-BE-NEXT:    uaddw v5.2d, v22.2d, v17.2s
+; CHECK-BE-NEXT:    st1 { v3.2d }, [x11]
+; CHECK-BE-NEXT:    uaddw v3.2d, v18.2d, v20.2s
+; CHECK-BE-NEXT:    uaddw v2.2d, v7.2d, v2.2s
 ; CHECK-BE-NEXT:    st1 { v4.2d }, [x16]
-; CHECK-BE-NEXT:    st1 { v6.2d }, [x12]
-; CHECK-BE-NEXT:    st1 { v3.2d }, [x13]
+; CHECK-BE-NEXT:    st1 { v5.2d }, [x13]
+; CHECK-BE-NEXT:    st1 { v3.2d }, [x14]
 ; CHECK-BE-NEXT:    st1 { v2.2d }, [x17]
 ; CHECK-BE-NEXT:    b.ne .LBB17_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
@@ -1813,14 +1812,14 @@ exit:
 define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {
 ; CHECK-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    mov w8, #128 ; =0x80
 ; CHECK-NEXT:    add x9, x1, #128
+; CHECK-NEXT:    add x10, x0, #16
 ; CHECK-NEXT:  LBB18_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x10, x0, x8
-; CHECK-NEXT:    add x8, x8, #16
-; CHECK-NEXT:    ldp q0, q1, [x10]
-; CHECK-NEXT:    cmp x8, #128
+; CHECK-NEXT:    ldp q0, q1, [x10, #-16]
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    add x10, x10, #16
 ; CHECK-NEXT:    ushll2.8h v2, v0, #0
 ; CHECK-NEXT:    ushll.8h v0, v0, #0
 ; CHECK-NEXT:    ushll2.8h v6, v1, #0
@@ -1863,18 +1862,18 @@ define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {
 ;
 ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
 ; CHECK-BE:       // %bb.0: // %entry
-; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:    mov w8, #128 // =0x80
 ; CHECK-BE-NEXT:    add x9, x1, #128
+; CHECK-BE-NEXT:    add x10, x0, #16
 ; CHECK-BE-NEXT:  .LBB18_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    add x10, x0, x8
-; CHECK-BE-NEXT:    sub x11, x9, #32
-; CHECK-BE-NEXT:    add x8, x8, #16
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x10]
-; CHECK-BE-NEXT:    add x10, x10, #16
-; CHECK-BE-NEXT:    cmp x8, #128
+; CHECK-BE-NEXT:    sub x11, x10, #16
 ; CHECK-BE-NEXT:    ld1 { v5.16b }, [x10]
-; CHECK-BE-NEXT:    sub x10, x9, #16
+; CHECK-BE-NEXT:    sub x12, x9, #32
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x11]
+; CHECK-BE-NEXT:    sub x11, x9, #16
+; CHECK-BE-NEXT:    subs x8, x8, #16
+; CHECK-BE-NEXT:    add x10, x10, #16
 ; CHECK-BE-NEXT:    ushll2 v1.8h, v0.16b, #0
 ; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    ushll2 v2.4s, v1.8h, #0
@@ -1885,54 +1884,54 @@ define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    ushll v2.2d, v2.2s, #0
 ; CHECK-BE-NEXT:    ushll2 v6.2d, v1.4s, #0
 ; CHECK-BE-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-BE-NEXT:    st1 { v4.2d }, [x10]
+; CHECK-BE-NEXT:    st1 { v4.2d }, [x11]
 ; CHECK-BE-NEXT:    ushll2 v4.2d, v3.4s, #0
 ; CHECK-BE-NEXT:    ushll v3.2d, v3.2s, #0
-; CHECK-BE-NEXT:    st1 { v2.2d }, [x11]
+; CHECK-BE-NEXT:    st1 { v2.2d }, [x12]
 ; CHECK-BE-NEXT:    ushll2 v2.8h, v5.16b, #0
-; CHECK-BE-NEXT:    sub x11, x9, #80
-; CHECK-BE-NEXT:    sub x10, x9, #48
-; CHECK-BE-NEXT:    st1 { v4.2d }, [x11]
+; CHECK-BE-NEXT:    sub x12, x9, #80
+; CHECK-BE-NEXT:    sub x11, x9, #48
+; CHECK-BE-NEXT:    st1 { v4.2d }, [x12]
 ; CHECK-BE-NEXT:    ushll v4.8h, v5.8b, #0
-; CHECK-BE-NEXT:    sub x11, x9, #64
+; CHECK-BE-NEXT:    sub x12, x9, #64
 ; CHECK-BE-NEXT:    ushll2 v5.4s, v2.8h, #0
-; CHECK-BE-NEXT:    st1 { v1.2d }, [x11]
-; CHECK-BE-NEXT:    sub x11, x9, #96
+; CHECK-BE-NEXT:    st1 { v1.2d }, [x12]
+; CHECK-BE-NEXT:    sub x12, x9, #96
 ; CHECK-BE-NEXT:    ushll2 v1.2d, v0.4s, #0
 ; CHECK-BE-NEXT:    ushll v2.4s, v2.4h, #0
 ; CHECK-BE-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-BE-NEXT:    st1 { v6.2d }, [x10]
-; CHECK-BE-NEXT:    sub x10, x9, #128
-; CHECK-BE-NEXT:    st1 { v3.2d }, [x11]
+; CHECK-BE-NEXT:    st1 { v6.2d }, [x11]
+; CHECK-BE-NEXT:    sub x11, x9, #128
+; CHECK-BE-NEXT:    st1 { v3.2d }, [x12]
 ; CHECK-BE-NEXT:    ushll2 v3.4s, v4.8h, #0
 ; CHECK-BE-NEXT:    ushll2 v6.2d, v5.4s, #0
-; CHECK-BE-NEXT:    sub x11, x9, #112
+; CHECK-BE-NEXT:    sub x12, x9, #112
 ; CHECK-BE-NEXT:    ushll v5.2d, v5.2s, #0
-; CHECK-BE-NEXT:    st1 { v0.2d }, [x10]
-; CHECK-BE-NEXT:    st1 { v1.2d }, [x11]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x11]
+; CHECK-BE-NEXT:    st1 { v1.2d }, [x12]
 ; CHECK-BE-NEXT:    ushll2 v1.2d, v2.4s, #0
-; CHECK-BE-NEXT:    add x10, x9, #112
+; CHECK-BE-NEXT:    add x11, x9, #112
 ; CHECK-BE-NEXT:    ushll v4.4s, v4.4h, #0
 ; CHECK-BE-NEXT:    ushll2 v0.2d, v3.4s, #0
-; CHECK-BE-NEXT:    st1 { v6.2d }, [x10]
-; CHECK-BE-NEXT:    add x10, x9, #96
+; CHECK-BE-NEXT:    st1 { v6.2d }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #96
 ; CHECK-BE-NEXT:    ushll v2.2d, v2.2s, #0
 ; CHECK-BE-NEXT:    ushll v3.2d, v3.2s, #0
-; CHECK-BE-NEXT:    st1 { v5.2d }, [x10]
-; CHECK-BE-NEXT:    add x10, x9, #80
-; CHECK-BE-NEXT:    st1 { v1.2d }, [x10]
-; CHECK-BE-NEXT:    add x10, x9, #48
+; CHECK-BE-NEXT:    st1 { v5.2d }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #80
+; CHECK-BE-NEXT:    st1 { v1.2d }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #48
 ; CHECK-BE-NEXT:    ushll2 v1.2d, v4.4s, #0
-; CHECK-BE-NEXT:    st1 { v0.2d }, [x10]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x11]
 ; CHECK-BE-NEXT:    ushll v0.2d, v4.2s, #0
-; CHECK-BE-NEXT:    add x10, x9, #64
-; CHECK-BE-NEXT:    st1 { v2.2d }, [x10]
-; CHECK-BE-NEXT:    add x10, x9, #32
-; CHECK-BE-NEXT:    st1 { v3.2d }, [x10]
-; CHECK-BE-NEXT:    add x10, x9, #16
+; CHECK-BE-NEXT:    add x11, x9, #64
+; CHECK-BE-NEXT:    st1 { v2.2d }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #32
+; CHECK-BE-NEXT:    st1 { v3.2d }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #16
 ; CHECK-BE-NEXT:    st1 { v0.2d }, [x9]
 ; CHECK-BE-NEXT:    add x9, x9, #128
-; CHECK-BE-NEXT:    st1 { v1.2d }, [x10]
+; CHECK-BE-NEXT:    st1 { v1.2d }, [x11]
 ; CHECK-BE-NEXT:    b.ne .LBB18_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
index 5c40a4ce13e31a1452239b2602135e307c3cc003..cb59121d6970837f434fa6749b883efee458eb12 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
@@ -135,31 +135,31 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s4
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 5, v1, vcc
-; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v0
-; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v0
+; GCN-NEXT:    v_sub_u32_e32 v1, vcc, 0, v0
+; GCN-NEXT:    v_max_i32_e32 v1, v0, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v1
 ; GCN-NEXT:    s_mov_b32 s4, 0xf4240
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
 ; GCN-NEXT:    v_mul_hi_u32 v2, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
 ; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0xf4240, v3
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v0
+; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v0
+; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %select = select i1 %cond, i32 ptrtoint (ptr addrspace(1) @gv to i32), i32 5
   %op = sdiv i32 1000000, %select
@@ -217,31 +217,31 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s4
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, 5, vcc
-; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v0
-; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v0
+; GCN-NEXT:    v_sub_u32_e32 v1, vcc, 0, v0
+; GCN-NEXT:    v_max_i32_e32 v1, v0, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v1
 ; GCN-NEXT:    s_mov_b32 s4, 0xf4240
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
 ; GCN-NEXT:    v_mul_hi_u32 v2, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
 ; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0xf4240, v3
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v0
+; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v0
+; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %select = select i1 %cond, i32 5, i32 ptrtoint (ptr addrspace(1) @gv to i32)
   %op = sdiv i32 1000000, %select
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 2ad28b8dd6ecf550349564518ab2b3b9f2fb88b8..8144fb7a3b64610561e22c08532a669d95c388d4 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -245,39 +245,36 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_ashr_i32 s8, s3, 31
-; GFX6-NEXT:    s_add_i32 s3, s3, s8
-; GFX6-NEXT:    s_xor_b32 s3, s3, s8
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX6-NEXT:    s_sub_i32 s4, 0, s3
-; GFX6-NEXT:    s_ashr_i32 s9, s2, 31
-; GFX6-NEXT:    s_add_i32 s2, s2, s9
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX6-NEXT:    s_xor_b32 s2, s2, s9
+; GFX6-NEXT:    s_abs_i32 s8, s3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT:    s_sub_i32 s4, 0, s8
 ; GFX6-NEXT:    s_mov_b32 s5, s1
+; GFX6-NEXT:    s_xor_b32 s1, s2, s3
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 31
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v0
 ; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_xor_b32 s0, s9, s8
+; GFX6-NEXT:    s_abs_i32 s0, s2
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s1, v0
-; GFX6-NEXT:    s_mul_i32 s1, s1, s3
-; GFX6-NEXT:    s_sub_i32 s1, s2, s1
-; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s8
+; GFX6-NEXT:    s_sub_i32 s0, s0, s2
+; GFX6-NEXT:    s_sub_i32 s2, s0, s8
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s8
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cselect_b32 s0, s2, s0
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s8
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v0, s0, v0
-; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s1, v0
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
@@ -286,16 +283,13 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s4, s3, 31
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT:    s_ashr_i32 s5, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s5
-; GFX9-NEXT:    s_xor_b32 s4, s5, s4
+; GFX9-NEXT:    s_abs_i32 s4, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX9-NEXT:    s_sub_i32 s5, 0, s4
+; GFX9-NEXT:    s_xor_b32 s3, s2, s3
+; GFX9-NEXT:    s_abs_i32 s2, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s5, 0, s3
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
@@ -303,18 +297,18 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX9-NEXT:    s_mul_hi_u32 s5, s6, s5
 ; GFX9-NEXT:    s_add_i32 s6, s6, s5
 ; GFX9-NEXT:    s_mul_hi_u32 s5, s2, s6
-; GFX9-NEXT:    s_mul_i32 s6, s5, s3
+; GFX9-NEXT:    s_mul_i32 s6, s5, s4
 ; GFX9-NEXT:    s_sub_i32 s2, s2, s6
 ; GFX9-NEXT:    s_add_i32 s7, s5, 1
-; GFX9-NEXT:    s_sub_i32 s6, s2, s3
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX9-NEXT:    s_sub_i32 s6, s2, s4
+; GFX9-NEXT:    s_cmp_ge_u32 s2, s4
 ; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
 ; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
 ; GFX9-NEXT:    s_add_i32 s6, s5, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX9-NEXT:    s_cmp_ge_u32 s2, s4
 ; GFX9-NEXT:    s_cselect_b32 s2, s6, s5
-; GFX9-NEXT:    s_xor_b32 s2, s2, s4
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX9-NEXT:    s_endpgm
@@ -366,37 +360,36 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX6-LABEL: srem_i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_ashr_i32 s4, s3, 31
-; GFX6-NEXT:    s_add_i32 s3, s3, s4
-; GFX6-NEXT:    s_xor_b32 s4, s3, s4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX6-NEXT:    s_sub_i32 s3, 0, s4
-; GFX6-NEXT:    s_ashr_i32 s5, s2, 31
-; GFX6-NEXT:    s_add_i32 s2, s2, s5
+; GFX6-NEXT:    s_abs_i32 s3, s3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT:    s_sub_i32 s4, 0, s3
+; GFX6-NEXT:    s_abs_i32 s8, s2
+; GFX6-NEXT:    s_mov_b32 s5, s1
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX6-NEXT:    s_xor_b32 s6, s2, s5
-; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
-; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT:    s_mov_b32 s4, s0
+; GFX6-NEXT:    s_ashr_i32 s0, s2, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s6, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX6-NEXT:    s_mul_i32 s7, s7, s4
-; GFX6-NEXT:    s_sub_i32 s6, s6, s7
-; GFX6-NEXT:    s_sub_i32 s7, s6, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s6, s4
-; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
-; GFX6-NEXT:    s_sub_i32 s7, s6, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s6, s4
-; GFX6-NEXT:    s_cselect_b32 s4, s7, s6
-; GFX6-NEXT:    s_xor_b32 s4, s4, s5
-; GFX6-NEXT:    s_sub_i32 s4, s4, s5
-; GFX6-NEXT:    v_mov_b32_e32 v0, s4
-; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX6-NEXT:    s_mul_i32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s1, s8, s1
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_xor_b32 s1, s1, s0
+; GFX6-NEXT:    s_sub_i32 s0, s1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: srem_i32:
@@ -404,15 +397,12 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s4, s3, 31
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
+; GFX9-NEXT:    s_abs_i32 s3, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
 ; GFX9-NEXT:    s_sub_i32 s5, 0, s3
 ; GFX9-NEXT:    s_ashr_i32 s4, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s4
+; GFX9-NEXT:    s_abs_i32 s2, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s4
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
@@ -1857,126 +1847,114 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX6-NEXT:    s_mov_b32 s19, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s18, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_ashr_i32 s2, s12, 31
-; GFX6-NEXT:    s_add_i32 s3, s12, s2
-; GFX6-NEXT:    s_xor_b32 s3, s3, s2
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX6-NEXT:    s_sub_i32 s4, 0, s3
+; GFX6-NEXT:    s_abs_i32 s2, s12
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX6-NEXT:    s_sub_i32 s3, 0, s2
+; GFX6-NEXT:    s_xor_b32 s4, s8, s12
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v0
-; GFX6-NEXT:    s_ashr_i32 s4, s8, 31
-; GFX6-NEXT:    s_add_i32 s5, s8, s4
-; GFX6-NEXT:    s_xor_b32 s5, s5, s4
+; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX6-NEXT:    s_abs_i32 s3, s8
+; GFX6-NEXT:    s_ashr_i32 s8, s4, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_xor_b32 s8, s4, s2
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s5, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX6-NEXT:    s_mul_i32 s2, s2, s3
-; GFX6-NEXT:    s_sub_i32 s2, s5, s2
-; GFX6-NEXT:    s_sub_i32 s4, s2, s3
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX6-NEXT:    s_mul_i32 s4, s4, s2
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_sub_i32 s4, s3, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX6-NEXT:    s_cselect_b32 s3, s4, s3
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
 ; GFX6-NEXT:    s_cselect_b64 s[2:3], -1, 0
-; GFX6-NEXT:    s_ashr_i32 s4, s13, 31
-; GFX6-NEXT:    s_add_i32 s5, s13, s4
-; GFX6-NEXT:    s_xor_b32 s5, s5, s4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
-; GFX6-NEXT:    s_sub_i32 s6, 0, s5
+; GFX6-NEXT:    s_abs_i32 s4, s13
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s4
+; GFX6-NEXT:    s_sub_i32 s5, 0, s4
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT:    s_xor_b32 s6, s9, s13
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
 ; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[2:3]
-; GFX6-NEXT:    v_xor_b32_e32 v0, s8, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT:    v_mul_lo_u32 v3, s6, v2
-; GFX6-NEXT:    s_ashr_i32 s6, s9, 31
-; GFX6-NEXT:    s_add_i32 s7, s9, s6
-; GFX6-NEXT:    s_xor_b32 s7, s7, s6
+; GFX6-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GFX6-NEXT:    v_mul_lo_u32 v3, s5, v2
+; GFX6-NEXT:    s_abs_i32 s5, s9
+; GFX6-NEXT:    s_ashr_i32 s9, s6, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v3
-; GFX6-NEXT:    s_xor_b32 s9, s6, s4
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_hi_u32 v2, s7, v2
-; GFX6-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX6-NEXT:    s_mul_i32 s4, s4, s5
-; GFX6-NEXT:    s_sub_i32 s4, s7, s4
-; GFX6-NEXT:    s_sub_i32 s6, s4, s5
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s5
+; GFX6-NEXT:    v_mul_hi_u32 v2, s5, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX6-NEXT:    s_mul_i32 s6, s6, s4
+; GFX6-NEXT:    s_sub_i32 s5, s5, s6
+; GFX6-NEXT:    s_sub_i32 s6, s5, s4
+; GFX6-NEXT:    s_cmp_ge_u32 s5, s4
 ; GFX6-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
-; GFX6-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s5
+; GFX6-NEXT:    s_cmp_ge_u32 s5, s4
 ; GFX6-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GFX6-NEXT:    s_ashr_i32 s6, s14, 31
-; GFX6-NEXT:    s_add_i32 s7, s14, s6
-; GFX6-NEXT:    s_xor_b32 s7, s7, s6
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s7
-; GFX6-NEXT:    s_sub_i32 s12, 0, s7
+; GFX6-NEXT:    s_abs_i32 s6, s14
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GFX6-NEXT:    s_sub_i32 s7, 0, s6
 ; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX6-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[4:5]
-; GFX6-NEXT:    v_xor_b32_e32 v2, s9, v2
 ; GFX6-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX6-NEXT:    v_mul_lo_u32 v5, s12, v4
-; GFX6-NEXT:    s_ashr_i32 s12, s10, 31
-; GFX6-NEXT:    s_add_i32 s10, s10, s12
-; GFX6-NEXT:    s_xor_b32 s10, s10, s12
+; GFX6-NEXT:    v_mul_lo_u32 v5, s7, v4
+; GFX6-NEXT:    s_abs_i32 s7, s10
+; GFX6-NEXT:    s_xor_b32 s10, s10, s14
+; GFX6-NEXT:    s_ashr_i32 s10, s10, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v5, v4, v5
-; GFX6-NEXT:    s_xor_b32 s12, s12, s6
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GFX6-NEXT:    v_mul_hi_u32 v4, s10, v4
-; GFX6-NEXT:    v_readfirstlane_b32 s6, v4
-; GFX6-NEXT:    s_mul_i32 s6, s6, s7
-; GFX6-NEXT:    s_sub_i32 s6, s10, s6
-; GFX6-NEXT:    s_sub_i32 s10, s6, s7
-; GFX6-NEXT:    s_cmp_ge_u32 s6, s7
+; GFX6-NEXT:    v_mul_hi_u32 v4, s7, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s12, v4
+; GFX6-NEXT:    s_mul_i32 s12, s12, s6
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_sub_i32 s12, s7, s6
+; GFX6-NEXT:    s_cmp_ge_u32 s7, s6
 ; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
-; GFX6-NEXT:    s_cselect_b32 s6, s10, s6
+; GFX6-NEXT:    s_cselect_b32 s7, s12, s7
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    s_cmp_ge_u32 s6, s7
+; GFX6-NEXT:    s_cmp_ge_u32 s7, s6
 ; GFX6-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; GFX6-NEXT:    s_ashr_i32 s10, s15, 31
-; GFX6-NEXT:    s_add_i32 s13, s15, s10
-; GFX6-NEXT:    s_xor_b32 s13, s13, s10
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v6, s13
-; GFX6-NEXT:    s_sub_i32 s0, 0, s13
+; GFX6-NEXT:    s_abs_i32 s12, s15
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v6, s12
+; GFX6-NEXT:    s_sub_i32 s0, 0, s12
 ; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v6
-; GFX6-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[6:7]
-; GFX6-NEXT:    v_xor_b32_e32 v4, s12, v4
+; GFX6-NEXT:    s_abs_i32 s1, s11
 ; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s9, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, s0, v3
-; GFX6-NEXT:    s_ashr_i32 s0, s11, 31
-; GFX6-NEXT:    s_add_i32 s1, s11, s0
-; GFX6-NEXT:    s_xor_b32 s1, s1, s0
-; GFX6-NEXT:    v_mul_hi_u32 v2, v3, v2
-; GFX6-NEXT:    s_xor_b32 s0, s0, s10
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_mul_hi_u32 v3, s1, v2
-; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s12, v4
-; GFX6-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX6-NEXT:    s_mul_i32 s2, s2, s13
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v4, v5, s[6:7]
+; GFX6-NEXT:    v_xor_b32_e32 v1, s9, v1
+; GFX6-NEXT:    v_mul_lo_u32 v2, s0, v6
+; GFX6-NEXT:    s_xor_b32 s0, s11, s15
+; GFX6-NEXT:    v_xor_b32_e32 v3, s10, v3
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 31
+; GFX6-NEXT:    v_mul_hi_u32 v2, v6, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s9, v1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; GFX6-NEXT:    v_mul_hi_u32 v4, s1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s10, v3
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v4
+; GFX6-NEXT:    s_mul_i32 s2, s2, s12
 ; GFX6-NEXT:    s_sub_i32 s1, s1, s2
-; GFX6-NEXT:    s_sub_i32 s2, s1, s13
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s13
+; GFX6-NEXT:    s_sub_i32 s2, s1, s12
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s12
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
 ; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s13
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s12
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX6-NEXT:    v_xor_b32_e32 v3, s0, v3
@@ -1990,16 +1968,13 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s2, s8, 31
-; GFX9-NEXT:    s_add_i32 s3, s8, s2
-; GFX9-NEXT:    s_xor_b32 s3, s3, s2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT:    s_ashr_i32 s8, s4, 31
-; GFX9-NEXT:    s_add_i32 s4, s4, s8
-; GFX9-NEXT:    s_xor_b32 s2, s8, s2
+; GFX9-NEXT:    s_abs_i32 s2, s8
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_xor_b32 s3, s4, s8
+; GFX9-NEXT:    s_sub_i32 s8, 0, s2
+; GFX9-NEXT:    s_abs_i32 s4, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s4, s8
-; GFX9-NEXT:    s_sub_i32 s8, 0, s3
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s12, v0
@@ -2007,108 +1982,99 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX9-NEXT:    s_mul_hi_u32 s8, s12, s8
 ; GFX9-NEXT:    s_add_i32 s12, s12, s8
 ; GFX9-NEXT:    s_mul_hi_u32 s8, s4, s12
-; GFX9-NEXT:    s_mul_i32 s12, s8, s3
+; GFX9-NEXT:    s_mul_i32 s12, s8, s2
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s12
 ; GFX9-NEXT:    s_add_i32 s13, s8, 1
-; GFX9-NEXT:    s_sub_i32 s12, s4, s3
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s3
+; GFX9-NEXT:    s_sub_i32 s12, s4, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
 ; GFX9-NEXT:    s_cselect_b32 s8, s13, s8
 ; GFX9-NEXT:    s_cselect_b32 s4, s12, s4
 ; GFX9-NEXT:    s_add_i32 s12, s8, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s12, s8
-; GFX9-NEXT:    s_ashr_i32 s4, s9, 31
-; GFX9-NEXT:    s_add_i32 s8, s9, s4
-; GFX9-NEXT:    s_xor_b32 s8, s8, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX9-NEXT:    s_ashr_i32 s9, s5, 31
-; GFX9-NEXT:    s_xor_b32 s3, s3, s2
-; GFX9-NEXT:    s_add_i32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s12, s8
+; GFX9-NEXT:    s_abs_i32 s4, s9
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    s_xor_b32 s8, s5, s9
+; GFX9-NEXT:    s_sub_i32 s9, 0, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s2, s3, s2
-; GFX9-NEXT:    s_xor_b32 s3, s5, s9
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_abs_i32 s5, s5
+; GFX9-NEXT:    s_ashr_i32 s8, s8, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s5, 0, s8
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
-; GFX9-NEXT:    s_mul_i32 s5, s5, s9
-; GFX9-NEXT:    s_mul_hi_u32 s5, s9, s5
-; GFX9-NEXT:    s_add_i32 s9, s9, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s9
-; GFX9-NEXT:    s_mul_i32 s9, s5, s8
-; GFX9-NEXT:    s_sub_i32 s3, s3, s9
-; GFX9-NEXT:    s_add_i32 s12, s5, 1
-; GFX9-NEXT:    s_sub_i32 s9, s3, s8
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    s_mul_i32 s9, s9, s3
+; GFX9-NEXT:    s_mul_hi_u32 s9, s3, s9
+; GFX9-NEXT:    s_add_i32 s3, s3, s9
+; GFX9-NEXT:    s_mul_hi_u32 s3, s5, s3
+; GFX9-NEXT:    s_mul_i32 s9, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, s5, s9
+; GFX9-NEXT:    s_add_i32 s12, s3, 1
+; GFX9-NEXT:    s_sub_i32 s9, s5, s4
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s12, s3
+; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX9-NEXT:    s_add_i32 s9, s3, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s9, s3
-; GFX9-NEXT:    s_add_i32 s9, s5, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s9, s5
-; GFX9-NEXT:    s_ashr_i32 s5, s10, 31
-; GFX9-NEXT:    s_add_i32 s8, s10, s5
-; GFX9-NEXT:    s_xor_b32 s8, s8, s5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX9-NEXT:    s_ashr_i32 s9, s6, 31
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
-; GFX9-NEXT:    s_add_i32 s6, s6, s9
+; GFX9-NEXT:    s_abs_i32 s4, s10
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX9-NEXT:    s_xor_b32 s3, s3, s8
+; GFX9-NEXT:    s_sub_i32 s9, 0, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, s8
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s5, s9, s5
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_xor_b32 s4, s6, s9
+; GFX9-NEXT:    s_xor_b32 s5, s6, s10
+; GFX9-NEXT:    s_abs_i32 s6, s6
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s6, 0, s8
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
-; GFX9-NEXT:    s_mul_i32 s6, s6, s9
-; GFX9-NEXT:    s_mul_hi_u32 s6, s9, s6
-; GFX9-NEXT:    s_add_i32 s9, s9, s6
-; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s9
-; GFX9-NEXT:    s_mul_i32 s9, s6, s8
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_add_i32 s10, s6, 1
-; GFX9-NEXT:    s_sub_i32 s9, s4, s8
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX9-NEXT:    s_add_i32 s9, s6, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s6
-; GFX9-NEXT:    s_ashr_i32 s6, s11, 31
-; GFX9-NEXT:    s_add_i32 s8, s11, s6
-; GFX9-NEXT:    s_xor_b32 s8, s8, s6
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    s_ashr_i32 s2, s7, 31
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX9-NEXT:    s_mul_i32 s9, s9, s8
+; GFX9-NEXT:    s_mul_hi_u32 s9, s8, s9
+; GFX9-NEXT:    s_add_i32 s8, s8, s9
+; GFX9-NEXT:    s_mul_hi_u32 s8, s6, s8
+; GFX9-NEXT:    s_mul_i32 s9, s8, s4
+; GFX9-NEXT:    s_sub_i32 s6, s6, s9
+; GFX9-NEXT:    s_add_i32 s10, s8, 1
+; GFX9-NEXT:    s_sub_i32 s9, s6, s4
+; GFX9-NEXT:    s_cmp_ge_u32 s6, s4
+; GFX9-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s9, s6
+; GFX9-NEXT:    s_add_i32 s9, s8, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s6, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s9, s8
+; GFX9-NEXT:    s_abs_i32 s6, s11
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
 ; GFX9-NEXT:    s_xor_b32 s4, s4, s5
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    s_add_i32 s7, s7, s2
-; GFX9-NEXT:    s_xor_b32 s6, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_xor_b32 s2, s7, s11
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX9-NEXT:    s_abs_i32 s3, s7
+; GFX9-NEXT:    s_sub_i32 s7, 0, s6
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s5
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    s_xor_b32 s2, s7, s2
-; GFX9-NEXT:    s_sub_i32 s5, 0, s8
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX9-NEXT:    s_mul_i32 s5, s5, s7
-; GFX9-NEXT:    s_mul_hi_u32 s5, s7, s5
-; GFX9-NEXT:    s_add_i32 s7, s7, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s2, s7
-; GFX9-NEXT:    s_mul_i32 s7, s5, s8
-; GFX9-NEXT:    s_sub_i32 s2, s2, s7
-; GFX9-NEXT:    s_add_i32 s9, s5, 1
-; GFX9-NEXT:    s_sub_i32 s7, s2, s8
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 31
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX9-NEXT:    s_mul_i32 s7, s7, s5
+; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s7
+; GFX9-NEXT:    s_add_i32 s5, s5, s7
+; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s5
+; GFX9-NEXT:    s_mul_i32 s7, s5, s6
+; GFX9-NEXT:    s_sub_i32 s3, s3, s7
+; GFX9-NEXT:    s_add_i32 s8, s5, 1
+; GFX9-NEXT:    s_sub_i32 s7, s3, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s7, s3
 ; GFX9-NEXT:    s_add_i32 s7, s5, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s5
-; GFX9-NEXT:    s_xor_b32 s2, s2, s6
-; GFX9-NEXT:    s_sub_i32 s2, s2, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s3, s7, s5
+; GFX9-NEXT:    s_xor_b32 s3, s3, s2
+; GFX9-NEXT:    s_sub_i32 s2, s3, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
 ; GFX9-NEXT:    s_endpgm
@@ -2279,116 +2245,104 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX6-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_ashr_i32 s2, s8, 31
-; GFX6-NEXT:    s_add_i32 s3, s8, s2
-; GFX6-NEXT:    s_xor_b32 s2, s3, s2
+; GFX6-NEXT:    s_abs_i32 s2, s8
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX6-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
-; GFX6-NEXT:    s_ashr_i32 s3, s4, 31
-; GFX6-NEXT:    s_add_i32 s4, s4, s3
-; GFX6-NEXT:    s_xor_b32 s4, s4, s3
+; GFX6-NEXT:    s_abs_i32 s3, s4
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s8, v0
 ; GFX6-NEXT:    s_mul_i32 s8, s8, s2
-; GFX6-NEXT:    s_sub_i32 s4, s4, s8
-; GFX6-NEXT:    s_sub_i32 s8, s4, s2
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s2
-; GFX6-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX6-NEXT:    s_sub_i32 s8, s4, s2
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s2
-; GFX6-NEXT:    s_cselect_b32 s2, s8, s4
-; GFX6-NEXT:    s_ashr_i32 s4, s9, 31
-; GFX6-NEXT:    s_add_i32 s8, s9, s4
-; GFX6-NEXT:    s_xor_b32 s4, s8, s4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX6-NEXT:    s_sub_i32 s8, 0, s4
-; GFX6-NEXT:    s_xor_b32 s2, s2, s3
-; GFX6-NEXT:    s_sub_i32 s9, s2, s3
+; GFX6-NEXT:    s_sub_i32 s3, s3, s8
+; GFX6-NEXT:    s_sub_i32 s8, s3, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX6-NEXT:    s_sub_i32 s8, s3, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s3
+; GFX6-NEXT:    s_abs_i32 s3, s9
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT:    s_sub_i32 s8, 0, s3
+; GFX6-NEXT:    s_xor_b32 s2, s2, s4
+; GFX6-NEXT:    s_sub_i32 s4, s2, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s8, v0
-; GFX6-NEXT:    s_ashr_i32 s8, s5, 31
-; GFX6-NEXT:    s_add_i32 s5, s5, s8
-; GFX6-NEXT:    s_xor_b32 s5, s5, s8
+; GFX6-NEXT:    s_abs_i32 s8, s5
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX6-NEXT:    s_mul_i32 s2, s2, s4
-; GFX6-NEXT:    s_sub_i32 s2, s5, s2
-; GFX6-NEXT:    s_sub_i32 s3, s2, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s4
-; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX6-NEXT:    s_sub_i32 s3, s2, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s4
-; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX6-NEXT:    s_ashr_i32 s3, s10, 31
-; GFX6-NEXT:    s_add_i32 s4, s10, s3
-; GFX6-NEXT:    s_xor_b32 s3, s4, s3
+; GFX6-NEXT:    s_mul_i32 s2, s2, s3
+; GFX6-NEXT:    s_sub_i32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX6-NEXT:    s_abs_i32 s3, s10
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX6-NEXT:    s_sub_i32 s4, 0, s3
-; GFX6-NEXT:    s_xor_b32 s2, s2, s8
+; GFX6-NEXT:    s_sub_i32 s8, 0, s3
+; GFX6-NEXT:    s_xor_b32 s2, s2, s5
+; GFX6-NEXT:    s_sub_i32 s5, s2, s5
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v0
-; GFX6-NEXT:    s_ashr_i32 s4, s6, 31
-; GFX6-NEXT:    s_add_i32 s5, s6, s4
-; GFX6-NEXT:    s_xor_b32 s5, s5, s4
+; GFX6-NEXT:    v_mul_lo_u32 v1, s8, v0
+; GFX6-NEXT:    s_abs_i32 s8, s6
+; GFX6-NEXT:    s_ashr_i32 s6, s6, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_sub_i32 s6, s2, s8
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX6-NEXT:    s_mul_i32 s2, s2, s3
-; GFX6-NEXT:    s_sub_i32 s2, s5, s2
-; GFX6-NEXT:    s_sub_i32 s5, s2, s3
+; GFX6-NEXT:    s_sub_i32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
 ; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
-; GFX6-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX6-NEXT:    s_sub_i32 s5, s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
 ; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
-; GFX6-NEXT:    s_cselect_b32 s5, s5, s2
-; GFX6-NEXT:    s_ashr_i32 s2, s11, 31
-; GFX6-NEXT:    s_add_i32 s3, s11, s2
-; GFX6-NEXT:    s_xor_b32 s8, s3, s2
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX6-NEXT:    s_sub_i32 s10, 0, s8
-; GFX6-NEXT:    s_xor_b32 s5, s5, s4
-; GFX6-NEXT:    s_sub_i32 s4, s5, s4
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s2
+; GFX6-NEXT:    s_abs_i32 s9, s11
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GFX6-NEXT:    s_sub_i32 s2, 0, s9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s9
-; GFX6-NEXT:    s_ashr_i32 s9, s7, 31
-; GFX6-NEXT:    s_add_i32 s7, s7, s9
-; GFX6-NEXT:    v_mul_lo_u32 v2, s10, v1
-; GFX6-NEXT:    s_xor_b32 s7, s7, s9
-; GFX6-NEXT:    v_mul_hi_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; GFX6-NEXT:    v_mul_hi_u32 v2, s7, v1
-; GFX6-NEXT:    v_mov_b32_e32 v1, s6
-; GFX6-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX6-NEXT:    s_mul_i32 s5, s5, s8
-; GFX6-NEXT:    s_sub_i32 s5, s7, s5
-; GFX6-NEXT:    s_sub_i32 s6, s5, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s8
-; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX6-NEXT:    s_sub_i32 s6, s5, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s8
-; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX6-NEXT:    s_xor_b32 s5, s5, s9
-; GFX6-NEXT:    s_sub_i32 s5, s5, s9
-; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_abs_i32 s4, s7
+; GFX6-NEXT:    v_mul_lo_u32 v1, s2, v2
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    s_ashr_i32 s5, s7, 31
+; GFX6-NEXT:    s_xor_b32 s7, s8, s6
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_mul_hi_u32 v2, s4, v2
+; GFX6-NEXT:    s_sub_i32 s6, s7, s6
+; GFX6-NEXT:    v_readfirstlane_b32 s7, v2
+; GFX6-NEXT:    s_mul_i32 s7, s7, s9
+; GFX6-NEXT:    s_sub_i32 s4, s4, s7
+; GFX6-NEXT:    s_sub_i32 s7, s4, s9
+; GFX6-NEXT:    s_cmp_ge_u32 s4, s9
+; GFX6-NEXT:    s_cselect_b32 s4, s7, s4
+; GFX6-NEXT:    s_sub_i32 s7, s4, s9
+; GFX6-NEXT:    s_cmp_ge_u32 s4, s9
+; GFX6-NEXT:    s_cselect_b32 s4, s7, s4
+; GFX6-NEXT:    s_xor_b32 s4, s4, s5
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    v_mov_b32_e32 v2, s6
+; GFX6-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
@@ -2398,15 +2352,12 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s2, s8, 31
-; GFX9-NEXT:    s_add_i32 s3, s8, s2
-; GFX9-NEXT:    s_xor_b32 s2, s3, s2
+; GFX9-NEXT:    s_abs_i32 s2, s8
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX9-NEXT:    s_sub_i32 s8, 0, s2
 ; GFX9-NEXT:    s_ashr_i32 s3, s4, 31
-; GFX9-NEXT:    s_add_i32 s4, s4, s3
+; GFX9-NEXT:    s_abs_i32 s4, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s4, s3
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s12, v0
@@ -2422,51 +2373,44 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX9-NEXT:    s_sub_i32 s8, s4, s2
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
 ; GFX9-NEXT:    s_cselect_b32 s2, s8, s4
-; GFX9-NEXT:    s_ashr_i32 s4, s9, 31
-; GFX9-NEXT:    s_add_i32 s8, s9, s4
-; GFX9-NEXT:    s_xor_b32 s4, s8, s4
+; GFX9-NEXT:    s_abs_i32 s4, s9
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT:    s_ashr_i32 s8, s5, 31
 ; GFX9-NEXT:    s_xor_b32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s5, s5, s8
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s9, 0, s4
 ; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s3, s5, s8
-; GFX9-NEXT:    s_sub_i32 s5, 0, s4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_ashr_i32 s8, s5, 31
+; GFX9-NEXT:    s_abs_i32 s5, s5
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
-; GFX9-NEXT:    s_mul_i32 s5, s5, s9
-; GFX9-NEXT:    s_mul_hi_u32 s5, s9, s5
-; GFX9-NEXT:    s_add_i32 s9, s9, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s9
-; GFX9-NEXT:    s_mul_i32 s5, s5, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    s_mul_i32 s9, s9, s3
+; GFX9-NEXT:    s_mul_hi_u32 s9, s3, s9
+; GFX9-NEXT:    s_add_i32 s3, s3, s9
+; GFX9-NEXT:    s_mul_hi_u32 s3, s5, s3
+; GFX9-NEXT:    s_mul_i32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s3, s5, s3
 ; GFX9-NEXT:    s_sub_i32 s5, s3, s4
 ; GFX9-NEXT:    s_cmp_ge_u32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
 ; GFX9-NEXT:    s_sub_i32 s5, s3, s4
 ; GFX9-NEXT:    s_cmp_ge_u32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_ashr_i32 s4, s10, 31
-; GFX9-NEXT:    s_add_i32 s5, s10, s4
-; GFX9-NEXT:    s_xor_b32 s4, s5, s4
+; GFX9-NEXT:    s_abs_i32 s4, s10
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
 ; GFX9-NEXT:    s_xor_b32 s3, s3, s8
+; GFX9-NEXT:    s_sub_i32 s9, 0, s4
 ; GFX9-NEXT:    s_sub_i32 s3, s3, s8
-; GFX9-NEXT:    s_sub_i32 s8, 0, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    s_ashr_i32 s5, s6, 31
-; GFX9-NEXT:    s_add_i32 s6, s6, s5
-; GFX9-NEXT:    s_xor_b32 s6, s6, s5
+; GFX9-NEXT:    s_abs_i32 s6, s6
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
-; GFX9-NEXT:    s_mul_i32 s8, s8, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s9, s8
-; GFX9-NEXT:    s_add_i32 s9, s9, s8
-; GFX9-NEXT:    s_mul_hi_u32 s8, s6, s9
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX9-NEXT:    s_mul_i32 s9, s9, s8
+; GFX9-NEXT:    s_mul_hi_u32 s9, s8, s9
+; GFX9-NEXT:    s_add_i32 s8, s8, s9
+; GFX9-NEXT:    s_mul_hi_u32 s8, s6, s8
 ; GFX9-NEXT:    s_mul_i32 s8, s8, s4
 ; GFX9-NEXT:    s_sub_i32 s6, s6, s8
 ; GFX9-NEXT:    s_sub_i32 s8, s6, s4
@@ -2475,36 +2419,34 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
 ; GFX9-NEXT:    s_sub_i32 s8, s6, s4
 ; GFX9-NEXT:    s_cmp_ge_u32 s6, s4
 ; GFX9-NEXT:    s_cselect_b32 s4, s8, s6
-; GFX9-NEXT:    s_ashr_i32 s6, s11, 31
-; GFX9-NEXT:    s_add_i32 s8, s11, s6
-; GFX9-NEXT:    s_xor_b32 s6, s8, s6
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT:    s_abs_i32 s6, s11
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GFX9-NEXT:    s_xor_b32 s4, s4, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    s_ashr_i32 s2, s7, 31
-; GFX9-NEXT:    s_xor_b32 s3, s4, s5
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    s_add_i32 s4, s7, s2
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_sub_i32 s5, 0, s6
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_abs_i32 s3, s7
+; GFX9-NEXT:    s_sub_i32 s7, 0, s6
 ; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    s_xor_b32 s4, s4, s2
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v2
-; GFX9-NEXT:    s_mul_i32 s5, s5, s7
-; GFX9-NEXT:    s_mul_hi_u32 s5, s7, s5
-; GFX9-NEXT:    s_add_i32 s7, s7, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, s7
-; GFX9-NEXT:    s_mul_i32 s5, s5, s6
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s5
-; GFX9-NEXT:    s_sub_i32 s5, s4, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_sub_i32 s5, s4, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_xor_b32 s4, s4, s2
-; GFX9-NEXT:    s_sub_i32 s2, s4, s2
-; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX9-NEXT:    s_mul_i32 s7, s7, s5
+; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s7
+; GFX9-NEXT:    s_add_i32 s5, s5, s7
+; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s5
+; GFX9-NEXT:    s_mul_i32 s5, s5, s6
+; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_sub_i32 s5, s3, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_sub_i32 s5, s3, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_xor_b32 s3, s3, s2
+; GFX9-NEXT:    s_sub_i32 s2, s3, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
 ; GFX9-NEXT:    s_endpgm
@@ -6538,71 +6480,65 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s6
-; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX6-NEXT:    s_add_i32 s2, s2, s3
-; GFX6-NEXT:    s_xor_b32 s2, s2, s3
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT:    s_sub_i32 s6, 0, s2
+; GFX6-NEXT:    s_abs_i32 s3, s2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT:    s_sub_i32 s6, 0, s3
+; GFX6-NEXT:    s_xor_b32 s2, s4, s2
 ; GFX6-NEXT:    s_lshl_b32 s7, 0x1000, s7
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
-; GFX6-NEXT:    s_ashr_i32 s6, s4, 31
-; GFX6-NEXT:    s_add_i32 s4, s4, s6
-; GFX6-NEXT:    s_xor_b32 s4, s4, s6
+; GFX6-NEXT:    s_abs_i32 s6, s4
+; GFX6-NEXT:    s_ashr_i32 s4, s2, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_xor_b32 s6, s6, s3
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX6-NEXT:    s_mul_i32 s3, s3, s2
-; GFX6-NEXT:    s_sub_i32 s3, s4, s3
-; GFX6-NEXT:    s_sub_i32 s4, s3, s2
+; GFX6-NEXT:    v_mul_hi_u32 v0, s6, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s3
+; GFX6-NEXT:    s_sub_i32 s2, s6, s2
+; GFX6-NEXT:    s_sub_i32 s6, s2, s3
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s6, s2
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    s_ashr_i32 s4, s7, 31
-; GFX6-NEXT:    s_add_i32 s7, s7, s4
-; GFX6-NEXT:    s_xor_b32 s7, s7, s4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s7
-; GFX6-NEXT:    s_sub_i32 s8, 0, s7
+; GFX6-NEXT:    s_abs_i32 s6, s7
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT:    s_sub_i32 s2, 0, s6
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v0, s6, v0
+; GFX6-NEXT:    s_xor_b32 s7, s5, s7
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_abs_i32 s5, s5
+; GFX6-NEXT:    v_xor_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    s_ashr_i32 s7, s7, 31
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mul_lo_u32 v3, s2, v2
 ; GFX6-NEXT:    s_mov_b32 s2, -1
-; GFX6-NEXT:    v_mul_lo_u32 v3, s8, v2
-; GFX6-NEXT:    s_ashr_i32 s8, s5, 31
-; GFX6-NEXT:    s_add_i32 s5, s5, s8
-; GFX6-NEXT:    s_xor_b32 s5, s5, s8
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v2, v3
-; GFX6-NEXT:    s_xor_b32 s4, s8, s4
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v1, s5, v1
-; GFX6-NEXT:    v_readfirstlane_b32 s6, v1
-; GFX6-NEXT:    s_mul_i32 s6, s6, s7
-; GFX6-NEXT:    s_sub_i32 s5, s5, s6
-; GFX6-NEXT:    s_sub_i32 s6, s5, s7
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX6-NEXT:    s_mul_i32 s4, s4, s6
+; GFX6-NEXT:    s_sub_i32 s4, s5, s4
+; GFX6-NEXT:    s_sub_i32 s5, s4, s6
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s7
+; GFX6-NEXT:    s_cmp_ge_u32 s4, s6
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s7
+; GFX6-NEXT:    s_cmp_ge_u32 s4, s6
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v1, s4, v1
-; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v1
+; GFX6-NEXT:    v_xor_b32_e32 v1, s7, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s7, v1
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
@@ -6613,65 +6549,59 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b32 s2, 0x1000, s6
-; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s2, s2, s3
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_abs_i32 s3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
 ; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s7
-; GFX9-NEXT:    s_ashr_i32 s7, s4, 31
-; GFX9-NEXT:    s_add_i32 s4, s4, s7
+; GFX9-NEXT:    s_abs_i32 s7, s4
+; GFX9-NEXT:    s_xor_b32 s2, s4, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s3, s7, s3
-; GFX9-NEXT:    s_xor_b32 s4, s4, s7
-; GFX9-NEXT:    s_sub_i32 s7, 0, s2
+; GFX9-NEXT:    s_sub_i32 s4, 0, s3
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
-; GFX9-NEXT:    s_mul_i32 s7, s7, s8
-; GFX9-NEXT:    s_mul_hi_u32 s7, s8, s7
-; GFX9-NEXT:    s_add_i32 s8, s8, s7
-; GFX9-NEXT:    s_mul_hi_u32 s7, s4, s8
-; GFX9-NEXT:    s_mul_i32 s8, s7, s2
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_add_i32 s9, s7, 1
-; GFX9-NEXT:    s_sub_i32 s8, s4, s2
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
-; GFX9-NEXT:    s_cselect_b32 s7, s9, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_add_i32 s8, s7, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s8, s7
-; GFX9-NEXT:    s_ashr_i32 s4, s6, 31
-; GFX9-NEXT:    s_add_i32 s6, s6, s4
-; GFX9-NEXT:    s_xor_b32 s6, s6, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT:    s_ashr_i32 s7, s5, 31
-; GFX9-NEXT:    s_xor_b32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s5, s5, s7
+; GFX9-NEXT:    s_mul_i32 s4, s4, s8
+; GFX9-NEXT:    s_mul_hi_u32 s4, s8, s4
+; GFX9-NEXT:    s_add_i32 s8, s8, s4
+; GFX9-NEXT:    s_mul_hi_u32 s4, s7, s8
+; GFX9-NEXT:    s_mul_i32 s8, s4, s3
+; GFX9-NEXT:    s_sub_i32 s7, s7, s8
+; GFX9-NEXT:    s_add_i32 s9, s4, 1
+; GFX9-NEXT:    s_sub_i32 s8, s7, s3
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX9-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX9-NEXT:    s_add_i32 s8, s4, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s4
+; GFX9-NEXT:    s_abs_i32 s4, s6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX9-NEXT:    s_xor_b32 s3, s3, s2
+; GFX9-NEXT:    s_sub_i32 s7, 0, s4
+; GFX9-NEXT:    s_sub_i32 s2, s3, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s7, s4
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s3, s5, s7
+; GFX9-NEXT:    s_xor_b32 s6, s5, s6
+; GFX9-NEXT:    s_abs_i32 s5, s5
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s5, 0, s6
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX9-NEXT:    s_mul_i32 s5, s5, s7
-; GFX9-NEXT:    s_mul_hi_u32 s5, s7, s5
-; GFX9-NEXT:    s_add_i32 s7, s7, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s7
-; GFX9-NEXT:    s_mul_i32 s7, s5, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_add_i32 s8, s5, 1
-; GFX9-NEXT:    s_sub_i32 s7, s3, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    s_mul_i32 s7, s7, s3
+; GFX9-NEXT:    s_mul_hi_u32 s7, s3, s7
+; GFX9-NEXT:    s_add_i32 s3, s3, s7
+; GFX9-NEXT:    s_mul_hi_u32 s3, s5, s3
+; GFX9-NEXT:    s_mul_i32 s7, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, s5, s7
+; GFX9-NEXT:    s_add_i32 s8, s3, 1
+; GFX9-NEXT:    s_sub_i32 s7, s5, s4
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX9-NEXT:    s_add_i32 s7, s3, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX9-NEXT:    s_add_i32 s7, s5, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s7, s5
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    s_xor_b32 s3, s3, s6
+; GFX9-NEXT:    s_sub_i32 s3, s3, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -7001,19 +6931,16 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s6
-; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX6-NEXT:    s_add_i32 s2, s2, s3
-; GFX6-NEXT:    s_xor_b32 s2, s2, s3
+; GFX6-NEXT:    s_abs_i32 s2, s2
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX6-NEXT:    s_sub_i32 s3, 0, s2
-; GFX6-NEXT:    s_ashr_i32 s6, s4, 31
+; GFX6-NEXT:    s_lshl_b32 s6, 0x1000, s7
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
-; GFX6-NEXT:    s_add_i32 s3, s4, s6
-; GFX6-NEXT:    s_xor_b32 s3, s3, s6
-; GFX6-NEXT:    s_lshl_b32 s4, 0x1000, s7
+; GFX6-NEXT:    s_abs_i32 s3, s4
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
@@ -7026,38 +6953,35 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX6-NEXT:    s_sub_i32 s7, s3, s2
 ; GFX6-NEXT:    s_cmp_ge_u32 s3, s2
 ; GFX6-NEXT:    s_cselect_b32 s7, s7, s3
-; GFX6-NEXT:    s_ashr_i32 s2, s4, 31
-; GFX6-NEXT:    s_add_i32 s4, s4, s2
-; GFX6-NEXT:    s_xor_b32 s4, s4, s2
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX6-NEXT:    s_sub_i32 s2, 0, s4
-; GFX6-NEXT:    s_ashr_i32 s8, s5, 31
-; GFX6-NEXT:    s_xor_b32 s7, s7, s6
+; GFX6-NEXT:    s_abs_i32 s6, s6
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT:    s_sub_i32 s2, 0, s6
+; GFX6-NEXT:    s_abs_i32 s8, s5
+; GFX6-NEXT:    s_xor_b32 s7, s7, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX6-NEXT:    s_sub_i32 s6, s7, s6
+; GFX6-NEXT:    s_sub_i32 s4, s7, s4
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT:    s_add_i32 s2, s5, s8
-; GFX6-NEXT:    s_xor_b32 s5, s2, s8
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX6-NEXT:    s_mul_i32 s7, s7, s4
-; GFX6-NEXT:    s_sub_i32 s5, s5, s7
-; GFX6-NEXT:    s_sub_i32 s7, s5, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s4
-; GFX6-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX6-NEXT:    s_sub_i32 s7, s5, s4
-; GFX6-NEXT:    s_cmp_ge_u32 s5, s4
-; GFX6-NEXT:    s_cselect_b32 s4, s7, s5
-; GFX6-NEXT:    s_xor_b32 s4, s4, s8
-; GFX6-NEXT:    s_sub_i32 s4, s4, s8
-; GFX6-NEXT:    v_mov_b32_e32 v0, s6
-; GFX6-NEXT:    v_mov_b32_e32 v1, s4
+; GFX6-NEXT:    s_mul_i32 s7, s7, s6
+; GFX6-NEXT:    s_sub_i32 s7, s8, s7
+; GFX6-NEXT:    s_sub_i32 s8, s7, s6
+; GFX6-NEXT:    s_cmp_ge_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX6-NEXT:    s_sub_i32 s8, s7, s6
+; GFX6-NEXT:    s_cmp_ge_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s8, s7
+; GFX6-NEXT:    s_xor_b32 s6, s6, s5
+; GFX6-NEXT:    s_sub_i32 s5, s6, s5
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
@@ -7068,16 +6992,13 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b32 s2, 0x1000, s6
-; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    s_abs_i32 s2, s2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX9-NEXT:    s_lshl_b32 s3, 0x1000, s7
 ; GFX9-NEXT:    s_sub_i32 s7, 0, s2
 ; GFX9-NEXT:    s_ashr_i32 s6, s4, 31
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_add_i32 s4, s4, s6
-; GFX9-NEXT:    s_xor_b32 s4, s4, s6
+; GFX9-NEXT:    s_abs_i32 s4, s4
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
@@ -7093,24 +7014,21 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX9-NEXT:    s_sub_i32 s7, s4, s2
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s2
 ; GFX9-NEXT:    s_cselect_b32 s2, s7, s4
-; GFX9-NEXT:    s_ashr_i32 s4, s3, 31
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
+; GFX9-NEXT:    s_abs_i32 s3, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
 ; GFX9-NEXT:    s_xor_b32 s2, s2, s6
+; GFX9-NEXT:    s_sub_i32 s7, 0, s3
 ; GFX9-NEXT:    s_sub_i32 s2, s2, s6
-; GFX9-NEXT:    s_sub_i32 s6, 0, s3
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    s_ashr_i32 s4, s5, 31
-; GFX9-NEXT:    s_add_i32 s5, s5, s4
-; GFX9-NEXT:    s_xor_b32 s5, s5, s4
+; GFX9-NEXT:    s_abs_i32 s5, s5
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX9-NEXT:    s_mul_i32 s6, s6, s7
-; GFX9-NEXT:    s_mul_hi_u32 s6, s7, s6
-; GFX9-NEXT:    s_add_i32 s7, s7, s6
-; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s7
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
+; GFX9-NEXT:    s_mul_i32 s7, s7, s6
+; GFX9-NEXT:    s_mul_hi_u32 s7, s6, s7
+; GFX9-NEXT:    s_add_i32 s6, s6, s7
+; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s6
 ; GFX9-NEXT:    s_mul_i32 s6, s6, s3
 ; GFX9-NEXT:    s_sub_i32 s5, s5, s6
 ; GFX9-NEXT:    s_sub_i32 s6, s5, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
index d6841d40f2313e022f29cdad162770a4c08c517c..bd61558905f6341910db297ddcc94a992d0d2421 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
@@ -52,9 +52,9 @@
 ; HSAOPT: [[LDZU:%[0-9]+]] = load i32, ptr addrspace(4) [[GEP1]], align 4, !range !2, !invariant.load !1
 ; HSAOPT: [[EXTRACTY:%[0-9]+]] = lshr i32 [[LDXY]], 16
 
-; HSAOPT: [[WORKITEM_ID_X:%[0-9]+]] = call i32 @llvm.amdgcn.workitem.id.x(), !range !3
-; HSAOPT: [[WORKITEM_ID_Y:%[0-9]+]] = call i32 @llvm.amdgcn.workitem.id.y(), !range !3
-; HSAOPT: [[WORKITEM_ID_Z:%[0-9]+]] = call i32 @llvm.amdgcn.workitem.id.z(), !range !3
+; HSAOPT: [[WORKITEM_ID_X:%[0-9]+]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; HSAOPT: [[WORKITEM_ID_Y:%[0-9]+]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; HSAOPT: [[WORKITEM_ID_Z:%[0-9]+]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
 
 ; HSAOPT: [[Y_SIZE_X_Z_SIZE:%[0-9]+]] = mul nuw nsw i32 [[EXTRACTY]], [[LDZU]]
 ; HSAOPT: [[YZ_X_XID:%[0-9]+]] = mul i32 [[Y_SIZE_X_Z_SIZE]], [[WORKITEM_ID_X]]
@@ -68,11 +68,11 @@
 ; HSAOPT: %arrayidx12 = getelementptr inbounds [5 x i32], ptr addrspace(3) [[LOCAL_GEP]], i32 0, i32 1
 
 
-; NOHSAOPT: call i32 @llvm.r600.read.local.size.y(), !range !1
-; NOHSAOPT: call i32 @llvm.r600.read.local.size.z(), !range !1
-; NOHSAOPT: call i32 @llvm.amdgcn.workitem.id.x(), !range !2
-; NOHSAOPT: call i32 @llvm.amdgcn.workitem.id.y(), !range !2
-; NOHSAOPT: call i32 @llvm.amdgcn.workitem.id.z(), !range !2
+; NOHSAOPT: call range(i32 0, 257) i32 @llvm.r600.read.local.size.y()
+; NOHSAOPT: call range(i32 0, 257) i32 @llvm.r600.read.local.size.z()
+; NOHSAOPT: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; NOHSAOPT: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; NOHSAOPT: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
 define amdgpu_kernel void @mova_same_clause(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %in) #0 {
 entry:
   %stack = alloca [5 x i32], align 4, addrspace(5)
@@ -533,8 +533,3 @@ attributes #1 = { nounwind "amdgpu-flat-work-group-size"="1,256" }
 !99 = !{i32 1, !"amdhsa_code_object_version", i32 400}
 
 ; HSAOPT: !1 = !{}
-; HSAOPT: !2 = !{i32 0, i32 257}
-; HSAOPT: !3 = !{i32 0, i32 256}
-
-; NOHSAOPT: !1 = !{i32 0, i32 257}
-; NOHSAOPT: !2 = !{i32 0, i32 256}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
index a62ea8fe4606a5a5c96babca2a8e34803e9d0697..84ea2beb8d04bd38f9f8b39646e4415ccbcfc1c9 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
@@ -22,75 +22,37 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
   ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE3]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[V_MOV_B32_e32_1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 64
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 128
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[COPY7]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[COPY7]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 72
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 144
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[COPY9]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[COPY9]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 80
@@ -108,6 +70,7 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE2]], [[COPY11]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
+  ; GCN-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
   ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 88
   ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
@@ -124,94 +87,95 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[COPY13:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY13]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 96
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 192
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY14:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY14]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY14]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN2]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY15:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET8]], [[REG_SEQUENCE2]], [[COPY15]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[REG_SEQUENCE2]], [[COPY15]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 104
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 208
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY16:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY16]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY16]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN2]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY17:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET8]], [[REG_SEQUENCE2]], [[COPY17]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[REG_SEQUENCE2]], [[COPY17]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 112
   ; GCN-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 224
   ; GCN-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY22:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY23:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY23]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY23]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY24:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY24]], [[REG_SEQUENCE2]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY24]], [[REG_SEQUENCE2]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY25:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY25]], [[REG_SEQUENCE2]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY25]], [[REG_SEQUENCE2]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY26:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY27:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY26]], [[REG_SEQUENCE2]], [[COPY27]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN12:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN13:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY26]], [[REG_SEQUENCE2]], [[COPY27]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY28:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 120
   ; GCN-NEXT:   [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 240
   ; GCN-NEXT:   [[COPY30:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY31:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY32:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY33:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY33]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY33]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY34:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY34]], [[REG_SEQUENCE2]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY34]], [[REG_SEQUENCE2]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY35:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY35]], [[REG_SEQUENCE2]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY35]], [[REG_SEQUENCE2]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY36:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY37:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY36]], [[REG_SEQUENCE2]], [[COPY37]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN12:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN13:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY36]], [[REG_SEQUENCE2]], [[COPY37]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY38:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY38]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
@@ -269,113 +233,63 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY58:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY58]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[COPY58]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY59:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 288
   ; GCN-NEXT:   [[COPY60:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY61:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY62:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY61]], [[S_LOAD_DWORDX4_IMM]], [[COPY62]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY61]], [[S_LOAD_DWORDX4_IMM]], [[COPY62]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY63:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY63]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[COPY63]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY64:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY64]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY64]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY65:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[COPY65]], [[REG_SEQUENCE2]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY65]], [[REG_SEQUENCE2]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN1]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY66:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY67:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY66]], [[REG_SEQUENCE2]], [[COPY67]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN12]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN13]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY66]], [[REG_SEQUENCE2]], [[COPY67]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY68:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY68]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[COPY68]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 152
   ; GCN-NEXT:   [[COPY69:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sreg_32 = S_MOV_B32 304
   ; GCN-NEXT:   [[COPY70:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY71:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY72:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY71]], [[S_LOAD_DWORDX4_IMM]], [[COPY72]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY71]], [[S_LOAD_DWORDX4_IMM]], [[COPY72]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY73:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY73]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[COPY73]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY74:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY74]], [[REG_SEQUENCE2]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY74]], [[REG_SEQUENCE2]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY75:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[COPY75]], [[REG_SEQUENCE2]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY75]], [[REG_SEQUENCE2]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY76:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY77:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY76]], [[REG_SEQUENCE2]], [[COPY77]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN12]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN13]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY76]], [[REG_SEQUENCE2]], [[COPY77]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   S_ENDPGM 0
 bb.0:
   %tmp0 = load <4 x i32>, ptr addrspace(6) %arg0, align 16, !invariant.load !0
   %tmp1 = load ptr addrspace(8), ptr addrspace(6) %arg0, align 16, !invariant.load !0
-  %buffer0 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 16, i1 false, i1 false) #0
-  %buffer1 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #0
-  %buffer2 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 1, i32 16, i1 false, i1 false) #0
-  %buffer3 = call nsz <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 16, i1 false, i1 false) #0
-
-  ; Insert inline asm to keep the different instruction types from being mixed.  This makes the output easier to read.
-  call void asm sideeffect "", "" ()
-
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer0, <4 x i32> %tmp0, i32 0, i32 32, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer2, <4 x i32> %tmp0, i32 1, i32 32, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %buffer3, <4 x i32> %tmp0, i32 %arg1, i32 32, i1 false, i1 false) #1
-
-  call void asm sideeffect "", "" ()
-
-  %buffer_format0 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 48, i1 false, i1 false) #0
-  %buffer_format1 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #0
-  %buffer_format2 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 1, i32 48, i1 false, i1 false) #0
-  %buffer_format3 = call nsz <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %tmp0, i32 %arg1, i32 48, i1 false, i1 false) #0
-
-  call void asm sideeffect "", "" ()
-
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format0, <4 x i32> %tmp0, i32 0, i32 64, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format2, <4 x i32> %tmp0, i32 1, i32 64, i1 false, i1 false) #1
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %buffer_format3, <4 x i32> %tmp0, i32 %arg1, i32 64, i1 false, i1 false) #1
-
-  call void asm sideeffect "", "" ()
-
-  %atomic_add0 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 80, i1 false) #2
-  %atomic_add1 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
-  %atomic_add2 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 1, i32 80, i1 false) #2
-  %atomic_add3 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 80, i1 false) #2
-
-  call void asm sideeffect "", "" ()
-
-  %atomic_cmpswap0 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 96, i1 false) #2
-  %atomic_cmpswap1 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
-  %atomic_cmpswap2 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 1, i32 96, i1 false) #2
-  %atomic_cmpswap3 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %arg1, i32 %arg1, <4 x i32> %tmp0, i32 %arg1, i32 96, i1 false) #2
-
-  call void asm sideeffect "", "" ()
-
-  %fadd1 = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 0, i32 112, i1 false) #2
-  %fadd2 = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 0, i32 %arg1, i1 false) #2
-  %fadd3 = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 1, i32 112, i1 false) #2
-  %fadd4 = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %tmp0, i32 %arg1, i32 112, i1 false) #2
-
-  call void asm sideeffect "", "" ()
 
-  ; rsrc, offset, soffset, cachepolicy
   %raw_buffer0 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 128, i32 0, i32 0) #0
   %raw_buffer1 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 64, i32 64, i32 0) #0
   %raw_buffer2 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 128, i32 0) #0
@@ -384,7 +298,6 @@ bb.0:
 
   call void asm sideeffect "", "" ()
 
-  ; rsrc, offset, soffset, cachepolicy
   %raw_ptr_buffer0 = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %tmp1, i32 128, i32 0, i32 0) #3
   %raw_ptr_buffer1 = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %tmp1, i32 64, i32 64, i32 0) #3
   %raw_ptr_buffer2 = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %tmp1, i32 0, i32 128, i32 0) #3
@@ -473,7 +386,6 @@ bb.0:
 
   call void asm sideeffect "", "" ()
 
-  ; rsrc, vindex, offset, soffset, cachepolicy
   %struct_buffer0 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 224, i32 0, i32 0) #0
   %struct_buffer1 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 112, i32 112, i32 0) #0
   %struct_buffer2 = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %tmp0, i32 0, i32 0, i32 224, i32 0) #0
@@ -593,13 +505,6 @@ bb.0:
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1
-declare i32 @llvm.amdgcn.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i1) #2
-declare i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32, i32, <4 x i32>, i32, i32, i1) #2
-declare float @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1) #2
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #0
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #0
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32) #2
@@ -630,7 +535,6 @@ declare void @llvm.amdgcn.struct.ptr.buffer.store.format.v4f32(<4 x float>, ptr
 
 
 attributes #0 = { nounwind readonly }
-attributes #1 = { nounwind writeonly }
 attributes #2 = { nounwind }
 attributes #3 = { nounwind memory(argmem: read) }
 attributes #4 = { nounwind memory(argmem: write) }
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-schedule.ll b/llvm/test/CodeGen/AMDGPU/buffer-schedule.ll
index bdc73e5b997205205f1c7755d65142b694a4e617..727863936096a72002d4a06de15a0a17fca0349b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-schedule.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-schedule.ll
@@ -4,46 +4,6 @@
 ; The buffer_loads and buffer_stores all access the same location. Check they do
 ; not get reordered by the scheduler.
 
-; GCN-LABEL: {{^}}_amdgpu_cs_main:
-; GCN: buffer_load_dword
-; GCN: buffer_store_dword
-; GCN: buffer_load_dword
-; GCN: buffer_store_dword
-; GCN: buffer_load_dword
-; GCN: buffer_store_dword
-; GCN: buffer_load_dword
-; GCN: buffer_store_dword
-
-; Function Attrs: nounwind
-define amdgpu_cs void @_amdgpu_cs_main(<3 x i32> inreg %arg3, <3 x i32> %arg5) {
-.entry:
-  %tmp9 = add <3 x i32> %arg3, %arg5
-  %tmp10 = extractelement <3 x i32> %tmp9, i32 0
-  %tmp11 = shl i32 %tmp10, 2
-  %tmp12 = inttoptr i64 undef to ptr addrspace(4)
-  %tmp13 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  %tmp14 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %tmp13, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp17 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  call void @llvm.amdgcn.buffer.store.f32(float %tmp14, <4 x i32> %tmp17, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp20 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  %tmp21 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %tmp20, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp22 = fadd reassoc nnan arcp contract float %tmp21, 1.000000e+00
-  call void @llvm.amdgcn.buffer.store.f32(float %tmp22, <4 x i32> %tmp20, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp25 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  %tmp26 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %tmp25, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp27 = fadd reassoc nnan arcp contract float %tmp26, 1.000000e+00
-  call void @llvm.amdgcn.buffer.store.f32(float %tmp27, <4 x i32> %tmp25, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp30 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  %tmp31 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %tmp30, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp32 = fadd reassoc nnan arcp contract float %tmp31, 1.000000e+00
-  call void @llvm.amdgcn.buffer.store.f32(float %tmp32, <4 x i32> %tmp30, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp35 = load <4 x i32>, ptr addrspace(4) %tmp12, align 16
-  %tmp36 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %tmp35, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  %tmp37 = fadd reassoc nnan arcp contract float %tmp36, 1.000000e+00
-  call void @llvm.amdgcn.buffer.store.f32(float %tmp37, <4 x i32> %tmp35, i32 0, i32 %tmp11, i1 false, i1 false) #0
-  ret void
-}
-
 ; GCN-LABEL: {{^}}test1:
 ; GCN: buffer_store_dword
 ; GCN: buffer_load_dword
@@ -84,10 +44,6 @@ define amdgpu_cs void @test1_ptrs_reorderable(ptr addrspace(8) inreg %buf, i32 %
 }
 
 
-declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #2
-
-declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #3
-
 declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32) #2
 
 declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32) #3
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2abcbbcdd1bc6e9ccadf2a656125469e5785bf3d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=r600-- -mcpu=redwood | FileCheck %s --check-prefixes=R600
+
+define amdgpu_kernel void @build_vector2 (ptr addrspace(1) %out) {
+; R600-LABEL: build_vector2:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.Y, literal.x,
+; R600-NEXT:    6(8.407791e-45), 0(0.000000e+00)
+; R600-NEXT:     MOV T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    5(7.006492e-45), 2(2.802597e-45)
+entry:
+  store <2 x i32> <i32 5, i32 6>, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @build_vector4 (ptr addrspace(1) %out) {
+; R600-LABEL: build_vector4:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 8, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, literal.x,
+; R600-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; R600-NEXT:     MOV * T0.Z, literal.x,
+; R600-NEXT:    7(9.809089e-45), 0(0.000000e+00)
+; R600-NEXT:     MOV * T0.Y, literal.x,
+; R600-NEXT:    6(8.407791e-45), 0(0.000000e+00)
+; R600-NEXT:     MOV T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    5(7.006492e-45), 2(2.802597e-45)
+entry:
+  store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @build_vector_v2i16 (ptr addrspace(1) %out) {
+; R600-LABEL: build_vector_v2i16:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV T4.X, literal.x,
+; R600-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; R600-NEXT:    393221(5.510200e-40), 2(2.802597e-45)
+entry:
+  store <2 x i16> <i16 5, i16 6>, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 %a) {
+; R600-LABEL: build_vector_v2i16_trunc:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR * T0.W, KC0[2].Z, literal.x,
+; R600-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; R600-NEXT:     OR_INT T4.X, PV.W, literal.x,
+; R600-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; R600-NEXT:    327680(4.591775e-40), 2(2.802597e-45)
+  %srl = lshr i32 %a, 16
+  %trunc = trunc i32 %srl to i16
+  %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0
+  %ins.1 = insertelement <2 x i16> %ins.0, i16 5, i32 1
+  store <2 x i16> %ins.1, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) {
+; R600-LABEL: build_v2i32_from_v4i16_shuffle:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 0, @10, KC0[], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 4, @11, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_16 T1.X, T0.X, 48, #3
+; R600-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV * T0.X, 0.0,
+; R600-NEXT:    ALU clause starting at 11:
+; R600-NEXT:     LSHL * T0.Y, T1.X, literal.x,
+; R600-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; R600-NEXT:     LSHL T0.X, T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    16(2.242078e-44), 2(2.802597e-45)
+entry:
+  %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> <i32 0, i32 2>
+  %zextended = zext <2 x i16> %shuf to <2 x i32>
+  %shifted = shl <2 x i32> %zextended, <i32 16, i32 16>
+  store <2 x i32> %shifted, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index 99755133f36d6a509c6804c2b069e3e61688837e..b26d15ed3a1c8a39f8961ca3d51cf50b26a02050 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -1,95 +1,256 @@
-; RUN: llc < %s -mtriple=r600 -mcpu=redwood | FileCheck %s --check-prefixes=R600,ALL
-; RUN: llc < %s -mtriple=amdgcn -verify-machineinstrs | FileCheck %s --check-prefixes=GFX6,GFX678,ALL
-; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefixes=GFX8,GFX678,ALL
-; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX10,GFX1011,ALL
-; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX11,GFX1011,ALL
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 | FileCheck %s --check-prefixes=GFX940,ALL
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefixes=GFX6,GCN
+; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=GFX8,GCN
+; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 | FileCheck %s --check-prefixes=GFX10,GCN
+; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-vopd=0 | FileCheck %s --check-prefixes=GFX11,GCN
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 | FileCheck %s --check-prefixes=GFX940,GCN
 
-; ALL-LABEL: {{^}}build_vector2:
-; R600: MOV
-; R600: MOV
-; R600-NOT: MOV
-; GFX678-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5
-; GFX678-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6
-; GFX1011-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5
-; GFX1011-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6
-; GFX678: buffer_store_dwordx2 v[[[X]]:[[Y]]]
-; GFX10: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX11: global_store_b64 v2, v[0:1], s[0:1]
 define amdgpu_kernel void @build_vector2 (ptr addrspace(1) %out) {
+; GFX6-LABEL: build_vector2:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, 5
+; GFX6-NEXT:    v_mov_b32_e32 v1, 6
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: build_vector2:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v1, 6
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: build_vector2:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 5
+; GFX10-NEXT:    v_mov_b32_e32 v1, 6
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: build_vector2:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 5
+; GFX11-NEXT:    v_mov_b32_e32 v1, 6
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX940-LABEL: build_vector2:
+; GFX940:       ; %bb.0: ; %entry
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 5
+; GFX940-NEXT:    v_mov_b32_e32 v1, 6
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
 entry:
   store <2 x i32> <i32 5, i32 6>, ptr addrspace(1) %out
   ret void
 }
 
-; ALL-LABEL: {{^}}build_vector4:
-; R600: MOV
-; R600: MOV
-; R600: MOV
-; R600: MOV
-; R600-NOT: MOV
-; GFX678-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5
-; GFX678-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6
-; GFX678-DAG: v_mov_b32_e32 v[[Z:[0-9]]], 7
-; GFX678-DAG: v_mov_b32_e32 v[[W:[0-9]]], 8
-; GFX1011-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5
-; GFX1011-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6
-; GFX1011-DAG: v_mov_b32_e32 v[[Z:[0-9]]], 7
-; GFX1011-DAG: v_mov_b32_e32 v[[W:[0-9]]], 8
-; GFX678: buffer_store_dwordx4 v[[[X]]:[[W]]]
-; GFX10: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX11: global_store_b128 v4, v[0:3], s[0:1]
 define amdgpu_kernel void @build_vector4 (ptr addrspace(1) %out) {
+; GFX6-LABEL: build_vector4:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, 5
+; GFX6-NEXT:    v_mov_b32_e32 v1, 6
+; GFX6-NEXT:    v_mov_b32_e32 v2, 7
+; GFX6-NEXT:    v_mov_b32_e32 v3, 8
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: build_vector4:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v0, 5
+; GFX8-NEXT:    v_mov_b32_e32 v1, 6
+; GFX8-NEXT:    v_mov_b32_e32 v2, 7
+; GFX8-NEXT:    v_mov_b32_e32 v3, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: build_vector4:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 5
+; GFX10-NEXT:    v_mov_b32_e32 v1, 6
+; GFX10-NEXT:    v_mov_b32_e32 v2, 7
+; GFX10-NEXT:    v_mov_b32_e32 v3, 8
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: build_vector4:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 5
+; GFX11-NEXT:    v_mov_b32_e32 v1, 6
+; GFX11-NEXT:    v_mov_b32_e32 v2, 7
+; GFX11-NEXT:    v_mov_b32_e32 v3, 8
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX940-LABEL: build_vector4:
+; GFX940:       ; %bb.0: ; %entry
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 5
+; GFX940-NEXT:    v_mov_b32_e32 v1, 6
+; GFX940-NEXT:    v_mov_b32_e32 v2, 7
+; GFX940-NEXT:    v_mov_b32_e32 v3, 8
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
 entry:
   store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, ptr addrspace(1) %out
   ret void
 }
 
-
-; ALL-LABEL: {{^}}build_vector_v2i16:
-; R600: MOV
-; R600-NOT: MOV
-; GFX678: s_mov_b32 s3, 0xf000
-; GFX678: s_mov_b32 s2, -1
-; GFX678: v_mov_b32_e32 v0, 0x60005
-; GFX678: s_waitcnt lgkmcnt(0)
-; GFX678: buffer_store_dword v0, off, s[0:3], 0
-; GFX1011: v_mov_b32_e32 v0, 0
-; GFX1011: v_mov_b32_e32 v1, 0x60005
-; GFX1011: s_waitcnt lgkmcnt(0)
-; GFX10: global_store_dword v0, v1, s[0:1]
-; GFX11: global_store_b32 v0, v1, s[0:1]
 define amdgpu_kernel void @build_vector_v2i16 (ptr addrspace(1) %out) {
+; GFX6-LABEL: build_vector_v2i16:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0x60005
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: build_vector_v2i16:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x60005
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: build_vector_v2i16:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0x60005
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: build_vector_v2i16:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0x60005
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX940-LABEL: build_vector_v2i16:
+; GFX940:       ; %bb.0: ; %entry
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x60005
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
 entry:
   store <2 x i16> <i16 5, i16 6>, ptr addrspace(1) %out
   ret void
 }
 
-; ALL-LABEL: {{^}}build_vector_v2i16_trunc:
-; R600: LSHR
-; R600: OR_INT
-; R600: LSHR
-; R600-NOT: MOV
-; GFX6: s_mov_b32 s3, 0xf000
-; GFX6: s_waitcnt lgkmcnt(0)
-; GFX6: v_alignbit_b32 v0, 5, s4, 16
-; GFX6: buffer_store_dword v0, off, s[0:3], 0
-; GFX8: s_mov_b32 s3, 0xf000
-; GFX8: s_mov_b32 s2, -1
-; GFX8: s_waitcnt lgkmcnt(0)
-; GFX8: s_lshr_b32 s4, s4, 16
-; GFX8: s_or_b32 s4, s4, 0x50000
-; GFX8: v_mov_b32_e32 v0, s4
-; GFX8: buffer_store_dword v0, off, s[0:3], 0
-; GFX1011: v_mov_b32_e32 v0, 0
-; GFX1011: s_waitcnt lgkmcnt(0)
-; GFX10: s_lshr_b32 s2, s2, 16
-; GFX10: s_pack_ll_b32_b16 s2, s2, 5
-; GFX11: s_pack_hl_b32_b16 s2, s2, 5
-; GFX1011: v_mov_b32_e32 v1, s2
-; GFX10: global_store_dword v0, v1, s[0:1]
-; GFX11: global_store_b32 v0, v1, s[0:1]
 define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 %a) {
+; GFX6-LABEL: build_vector_v2i16_trunc:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, 5, s4, 16
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: build_vector_v2i16_trunc:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, 0x50000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: build_vector_v2i16_trunc:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x8
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, 5
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: build_vector_v2i16_trunc:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_pack_hl_b32_b16 s2, s2, 5
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX940-LABEL: build_vector_v2i16_trunc:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    s_pack_ll_b32_b16 s0, s0, 5
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
   %srl = lshr i32 %a, 16
   %trunc = trunc i32 %srl to i16
   %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0
@@ -98,54 +259,33 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
   ret void
 }
 
-; R600-LABEL: build_v2i32_from_v4i16_shuffle:
-; R600:       ; %bb.0: ; %entry
-; R600-NEXT:    ALU 0, @10, KC0[], KC1[]
-; R600-NEXT:    TEX 1 @6
-; R600-NEXT:    ALU 4, @11, KC0[CB0:0-32], KC1[]
-; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
-; R600-NEXT:    CF_END
-; R600-NEXT:    PAD
-; R600-NEXT:    Fetch clause starting at 6:
-; R600-NEXT:     VTX_READ_16 T1.X, T0.X, 48, #3
-; R600-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
-; R600-NEXT:    ALU clause starting at 10:
-; R600-NEXT:     MOV * T0.X, 0.0,
-; R600-NEXT:    ALU clause starting at 11:
-; R600-NEXT:     LSHL * T0.Y, T1.X, literal.x,
-; R600-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; R600-NEXT:     LSHL T0.X, T0.X, literal.x,
-; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
-; R600-NEXT:    16(2.242078e-44), 2(2.802597e-45)
-;
+define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) {
 ; GFX6-LABEL: build_v2i32_from_v4i16_shuffle:
 ; GFX6:       ; %bb.0: ; %entry
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
 ; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_mov_b32 s4, s0
 ; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    v_mov_b32_e32 v0, s2
-; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    s_lshl_b32 s0, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: build_v2i32_from_v4i16_shuffle:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX8-NEXT:    s_mov_b32 s7, 0xf000
-; GFX8-NEXT:    s_mov_b32 s6, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, s0
-; GFX8-NEXT:    s_mov_b32 s5, s1
-; GFX8-NEXT:    s_lshl_b32 s0, s3, 16
-; GFX8-NEXT:    s_lshl_b32 s1, s2, 16
-; GFX8-NEXT:    v_mov_b32_e32 v0, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: build_v2i32_from_v4i16_shuffle:
@@ -185,7 +325,6 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
 ; GFX940-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
 ; GFX940-NEXT:    s_endpgm
-define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) {
 entry:
   %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %zextended = zext <2 x i16> %shuf to <2 x i32>
@@ -193,3 +332,5 @@ entry:
   store <2 x i32> %shifted, ptr addrspace(1) %out
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
index 4d8687b141a79af005d73fba3226dad234baed1d..5bbea7ecf3f2d524cded36de60b461e38caea2a6 100644
--- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll
+++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
@@ -575,34 +575,33 @@ define i32 @sdiv32(i32 %a, i32 %b) {
 ; GFX9-LABEL: sdiv32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v1
-; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_sub_u32_e32 v2, 0, v1
+; GFX9-NEXT:    v_max_i32_e32 v2, v1, v2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v2
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v0
+; GFX9-NEXT:    v_max_i32_e32 v5, v0, v5
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v5
-; GFX9-NEXT:    v_xor_b32_e32 v2, v5, v2
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
 ; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, v3
-; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v4, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_add_u32_e32 v4, 1, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_mul_hi_u32 v3, v5, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v4, v5, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = sdiv i32 %a, %b
   ret i32 %d
@@ -640,31 +639,30 @@ define i32 @srem32(i32 %a, i32 %b) {
 ; GFX9-LABEL: srem32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, 0, v1
+; GFX9-NEXT:    v_max_i32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v1
 ; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
-; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v0
+; GFX9-NEXT:    v_max_i32_e32 v4, v0, v4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v2
 ; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GFX9-NEXT:    v_mul_hi_u32 v2, v4, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v2, v4, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v2, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, v2, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = srem i32 %a, %b
   ret i32 %d
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index 0a0179e866cd335291998ac5f92dc0f232ba3165..84bd9b6f6c5d48cf919871888879de687c8b53b1 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1489,9 +1489,8 @@ define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(ptr addrspace(1) %ou
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
 ; SI-NEXT:    buffer_load_ushort v1, v[1:2], s[4:7], 0 addr64
-; SI-NEXT:    v_cvt_f32_f16_e64 v3, s6 clamp
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, 0
 ; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_add_f32_e32 v1, 1.0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
index c347437c30828369697ebfcb937db2210ad22319..40d77a7f51e9d3136d858880a1c5062186c26394 100644
--- a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
+++ b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
@@ -105,6 +105,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1103 < %s | FileCheck --check-prefixes=GFX1103 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1150 < %s | FileCheck --check-prefixes=GFX1150 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 < %s | FileCheck --check-prefixes=GFX1151 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1152 < %s | FileCheck --check-prefixes=GFX1152 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX1200 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1201 < %s | FileCheck --check-prefixes=GFX1201 %s
 
@@ -201,6 +202,7 @@
 ; GFX1103: .amdgcn_target "amdgcn-amd-amdhsa--gfx1103"
 ; GFX1150: .amdgcn_target "amdgcn-amd-amdhsa--gfx1150"
 ; GFX1151: .amdgcn_target "amdgcn-amd-amdhsa--gfx1151"
+; GFX1152: .amdgcn_target "amdgcn-amd-amdhsa--gfx1152"
 ; GFX1200: .amdgcn_target "amdgcn-amd-amdhsa--gfx1200"
 ; GFX1201: .amdgcn_target "amdgcn-amd-amdhsa--gfx1201"
 
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 7d8eba1e870804a98c3fd69278731ce2b392ab83..f0ab3a5342e019bf71225d6bfaaae7423c08a0ed 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -9,26 +9,28 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-LABEL: v_sdiv_i128_vv:
 ; GFX9:       ; %bb.0: ; %_udiv-special-cases
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, 0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v16, 31, v3
-; GFX9-NEXT:    v_xor_b32_e32 v0, v16, v0
-; GFX9-NEXT:    v_xor_b32_e32 v1, v16, v1
-; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v0, v16
-; GFX9-NEXT:    v_xor_b32_e32 v2, v16, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v1, v16, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, v3, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, v2, v10, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, 0, v4
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[6:7]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v17, 31, v7
-; GFX9-NEXT:    v_xor_b32_e32 v3, v16, v3
-; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, v2, v16, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, v3, v16, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v3, v17, v4
-; GFX9-NEXT:    v_xor_b32_e32 v2, v17, v5
-; GFX9-NEXT:    v_sub_co_u32_e32 v20, vcc, v3, v17
-; GFX9-NEXT:    v_xor_b32_e32 v0, v17, v6
-; GFX9-NEXT:    v_subb_co_u32_e32 v21, vcc, v2, v17, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, v17, v7
-; GFX9-NEXT:    v_subb_co_u32_e32 v0, vcc, v0, v17, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v17, vcc
-; GFX9-NEXT:    v_or_b32_e32 v3, v21, v1
-; GFX9-NEXT:    v_or_b32_e32 v2, v20, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v20, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v21, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT:    v_or_b32_e32 v3, v20, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v21, v0
 ; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
 ; GFX9-NEXT:    v_or_b32_e32 v3, v9, v11
 ; GFX9-NEXT:    v_or_b32_e32 v2, v8, v10
@@ -37,35 +39,35 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 32, v2
 ; GFX9-NEXT:    v_ffbh_u32_e32 v3, v1
 ; GFX9-NEXT:    v_min_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_ffbh_u32_e32 v3, v20
+; GFX9-NEXT:    v_ffbh_u32_e32 v3, v21
 ; GFX9-NEXT:    v_add_u32_e32 v3, 32, v3
-; GFX9-NEXT:    v_ffbh_u32_e32 v4, v21
+; GFX9-NEXT:    v_ffbh_u32_e32 v4, v20
 ; GFX9-NEXT:    v_min_u32_e32 v3, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, 64, v3
 ; GFX9-NEXT:    v_addc_co_u32_e64 v4, s[6:7], 0, 0, vcc
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX9-NEXT:    v_ffbh_u32_e32 v5, v11
+; GFX9-NEXT:    v_ffbh_u32_e32 v6, v11
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
 ; GFX9-NEXT:    v_ffbh_u32_e32 v3, v10
 ; GFX9-NEXT:    v_add_u32_e32 v3, 32, v3
-; GFX9-NEXT:    v_min_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_ffbh_u32_e32 v5, v8
-; GFX9-NEXT:    v_add_u32_e32 v5, 32, v5
-; GFX9-NEXT:    v_ffbh_u32_e32 v6, v9
-; GFX9-NEXT:    v_min_u32_e32 v5, v5, v6
+; GFX9-NEXT:    v_min_u32_e32 v3, v3, v6
+; GFX9-NEXT:    v_ffbh_u32_e32 v6, v8
+; GFX9-NEXT:    v_add_u32_e32 v6, 32, v6
+; GFX9-NEXT:    v_ffbh_u32_e32 v7, v9
+; GFX9-NEXT:    v_min_u32_e32 v6, v6, v7
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, 64, v5
-; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 64, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v7, s[6:7], 0, 0, vcc
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0x7f
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
-; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v2, v3
-; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v4, v6, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, 0, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v2, v3
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v4, v7, vcc
 ; GFX9-NEXT:    v_subbrev_co_u32_e32 v4, vcc, 0, v5, vcc
 ; GFX9-NEXT:    v_subbrev_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0x7f
 ; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
 ; GFX9-NEXT:    v_mov_b32_e32 v18, v16
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
@@ -138,8 +140,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v7, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v6, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, -1, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, -1, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, -1, v21
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, -1, v20, vcc
 ; GFX9-NEXT:    v_addc_co_u32_e32 v28, vcc, -1, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v12, 0
@@ -164,9 +166,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_subb_co_u32_e32 v14, vcc, v28, v10, vcc
 ; GFX9-NEXT:    v_subb_co_u32_e32 v14, vcc, v29, v11, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v30, 31, v14
-; GFX9-NEXT:    v_and_b32_e32 v14, v30, v20
-; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v8, v14
 ; GFX9-NEXT:    v_and_b32_e32 v14, v30, v21
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v8, v14
+; GFX9-NEXT:    v_and_b32_e32 v14, v30, v20
 ; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v9, v14, vcc
 ; GFX9-NEXT:    v_and_b32_e32 v14, v30, v0
 ; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, v10, v14, vcc
@@ -218,238 +220,244 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0:       ; %bb.0: ; %_udiv-special-cases
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-O0-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $vgpr8 : SGPR spill to VGPR lane
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v4
+; GFX9-O0-NEXT:    v_mov_b32_e32 v21, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v4
 ; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v0
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v2
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
+; GFX9-O0-NEXT:    ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v22, v7
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v3
+; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v5
+; GFX9-O0-NEXT:    ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v2
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
+; GFX9-O0-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GFX9-O0-NEXT:    s_mov_b32 s4, 63
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v4
-; GFX9-O0-NEXT:    v_ashrrev_i64 v[13:14], s4, v[11:12]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v2
-; GFX9-O0-NEXT:    v_ashrrev_i64 v[11:12], s4, v[11:12]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v14
-; GFX9-O0-NEXT:    v_xor_b32_e64 v1, v6, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v4
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v13
-; GFX9-O0-NEXT:    v_xor_b32_e64 v13, v4, v5
-; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v10
-; GFX9-O0-NEXT:    v_xor_b32_e64 v1, v6, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
-; GFX9-O0-NEXT:    v_xor_b32_e64 v15, v4, v5
-; GFX9-O0-NEXT:    ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v15
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v16
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v14
-; GFX9-O0-NEXT:    v_sub_co_u32_e32 v9, vcc, v9, v4
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v6, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v13, vcc, v10, v4, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v4
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v10
+; GFX9-O0-NEXT:    v_mov_b32_e32 v20, v11
+; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 0
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 1
+; GFX9-O0-NEXT:    s_mov_b32 s10, s6
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s10, 2
+; GFX9-O0-NEXT:    s_mov_b32 s11, s7
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s11, 3
+; GFX9-O0-NEXT:    v_sub_co_u32_e32 v6, vcc, s10, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v4, vcc, v1, v3, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s10
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v14, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v20, vcc
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v4
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v7
+; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_lt_i64_e64 s[4:5], v[10:11], s[4:5]
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v4, v3, v4, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v6
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v4
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v6
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v2, v20, v1, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v5
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v1, v14, v1, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v12
-; GFX9-O0-NEXT:    v_xor_b32_e64 v1, v5, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v11
-; GFX9-O0-NEXT:    v_xor_b32_e64 v11, v3, v2
-; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v8
-; GFX9-O0-NEXT:    v_xor_b32_e64 v1, v5, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v7
-; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v3, v2
-; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v7
-; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 killed $vgpr7_vgpr8 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v12
-; GFX9-O0-NEXT:    v_sub_co_u32_e32 v1, vcc, v1, v3
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v8, vcc, v8, v5, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v11, vcc, v7, v3, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v7, vcc, v2, v5, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v8
+; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v21
+; GFX9-O0-NEXT:    v_mov_b32_e32 v15, v22
+; GFX9-O0-NEXT:    v_sub_co_u32_e32 v18, vcc, s10, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v10, vcc, v8, v9, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, s10
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v11, vcc, v8, v13, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v8, vcc, v8, v15, vcc
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v8
+; GFX9-O0-NEXT:    ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v19, v10
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v19
+; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_lt_i64_e64 s[4:5], v[21:22], s[4:5]
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, v9, v10, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v18
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[4:5]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v19, v10
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v12
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v15, v8, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v11
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v8, v13, v8, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v7
-; GFX9-O0-NEXT:    v_xor_b32_e64 v5, v5, v6
-; GFX9-O0-NEXT:    v_xor_b32_e64 v3, v3, v4
-; GFX9-O0-NEXT:    ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v3
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v9
+; GFX9-O0-NEXT:    v_xor_b32_e64 v15, v15, v20
+; GFX9-O0-NEXT:    v_xor_b32_e64 v13, v13, v14
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v15
+; GFX9-O0-NEXT:    s_mov_b32 s4, 63
+; GFX9-O0-NEXT:    v_ashrrev_i64 v[13:14], s4, v[13:14]
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v11
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v18
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v19
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v14
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v5
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v17
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v2
-; GFX9-O0-NEXT:    v_or_b32_e64 v3, v8, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
-; GFX9-O0-NEXT:    v_or_b32_e64 v1, v5, v6
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 0
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 0
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 1
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v14
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT:    v_or_b32_e64 v15, v4, v2
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_or_b32_e64 v9, v3, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v15
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v19
+; GFX9-O0-NEXT:    v_or_b32_e64 v15, v13, v14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v11
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v18
+; GFX9-O0-NEXT:    v_or_b32_e64 v13, v13, v14
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v15
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[4:5], v[13:14], s[6:7]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT:    v_or_b32_e64 v15, v13, v14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT:    v_or_b32_e64 v13, v13, v14
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v15
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[13:14], s[6:7]
 ; GFX9-O0-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v6, v6
-; GFX9-O0-NEXT:    s_mov_b32 s9, 32
-; GFX9-O0-NEXT:    v_add_u32_e64 v6, v6, s9
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v7, v7
-; GFX9-O0-NEXT:    v_min_u32_e64 v6, v6, v7
-; GFX9-O0-NEXT:    s_mov_b32 s8, 0
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v7
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v5
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[11:12], s[8:9]
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v8, v8
-; GFX9-O0-NEXT:    v_min_u32_e64 v15, v5, v8
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v5
-; GFX9-O0-NEXT:    s_mov_b64 s[10:11], 64
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v15
-; GFX9-O0-NEXT:    s_mov_b32 s12, s10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v16
-; GFX9-O0-NEXT:    s_mov_b32 s14, s11
-; GFX9-O0-NEXT:    v_add_co_u32_e64 v8, s[12:13], v8, s12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s14
-; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13]
+; GFX9-O0-NEXT:    s_mov_b32 s13, 32
+; GFX9-O0-NEXT:    v_add_u32_e64 v8, v8, s13
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v9, v9
+; GFX9-O0-NEXT:    v_min_u32_e64 v8, v8, v9
+; GFX9-O0-NEXT:    s_mov_b32 s12, 0
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
-; GFX9-O0-NEXT:    s_mov_b64 s[12:13], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13]
-; GFX9-O0-NEXT:    v_cndmask_b32_e64 v5, v5, v10, s[12:13]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v8
-; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v6, v7, s[12:13]
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v11
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v9
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v7, v7
+; GFX9-O0-NEXT:    v_add_u32_e64 v7, v7, s13
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v10, v10
+; GFX9-O0-NEXT:    v_min_u32_e64 v13, v7, v10
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, s12
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v7
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], 64
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v13
+; GFX9-O0-NEXT:    s_mov_b32 s16, s14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v14
+; GFX9-O0-NEXT:    s_mov_b32 s18, s15
+; GFX9-O0-NEXT:    v_add_co_u32_e64 v10, s[16:17], v10, s16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s18
+; GFX9-O0-NEXT:    v_addc_co_u32_e64 v7, s[16:17], v7, v11, s[16:17]
+; GFX9-O0-NEXT:    ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v11
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v12, s[8:9]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v10
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v8, v9, s[8:9]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v7
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[5:6], s[8:9]
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v1
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s13
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v6, v2
 ; GFX9-O0-NEXT:    v_min_u32_e64 v6, v5, v6
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v5
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v7
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v3
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s13
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v11, v4
-; GFX9-O0-NEXT:    v_min_u32_e64 v15, v5, v11
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v15
-; GFX9-O0-NEXT:    s_mov_b32 s8, s10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v16
-; GFX9-O0-NEXT:    s_mov_b32 s10, s11
-; GFX9-O0-NEXT:    v_add_co_u32_e64 v11, s[8:9], v11, s8
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s10
-; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9]
+; GFX9-O0-NEXT:    v_min_u32_e64 v12, v5, v11
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr13
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s12
+; GFX9-O0-NEXT:    ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v12
+; GFX9-O0-NEXT:    s_mov_b32 s12, s14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v13
+; GFX9-O0-NEXT:    s_mov_b32 s14, s15
+; GFX9-O0-NEXT:    v_add_co_u32_e64 v11, s[12:13], v11, s12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s14
+; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[12:13], v5, v12, s[12:13]
 ; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v5
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v12
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[8:9]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v11
@@ -462,8 +470,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v10
-; GFX9-O0-NEXT:    s_mov_b32 s10, s6
-; GFX9-O0-NEXT:    s_mov_b32 s11, s7
 ; GFX9-O0-NEXT:    v_sub_co_u32_e32 v5, vcc, v5, v8
 ; GFX9-O0-NEXT:    v_subb_co_u32_e32 v9, vcc, v6, v7, vcc
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, s10
@@ -546,75 +552,75 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[4:5], exec
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s4, 2
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s5, 3
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s4, 4
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s5, 5
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execz .LBB0_3
 ; GFX9-O0-NEXT:    s_branch .LBB0_8
 ; GFX9-O0-NEXT:  .LBB0_1: ; %Flow
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v0, 4
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v0, 5
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v0, 6
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v0, 7
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:  ; %bb.2: ; %Flow
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_5
 ; GFX9-O0-NEXT:  .LBB0_3: ; %Flow2
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v4, 2
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v4, 3
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v4, 4
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v4, 5
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_9
 ; GFX9-O0-NEXT:  .LBB0_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 1
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-O0-NEXT:    v_lshlrev_b64 v[2:3], s4, v[0:1]
@@ -647,67 +653,67 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_3
 ; GFX9-O0-NEXT:  .LBB0_5: ; %Flow1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v8, 6
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v8, 7
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v8, 8
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v8, 9
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_4
 ; GFX9-O0-NEXT:  .LBB0_6: ; %udiv-do-while
 ; GFX9-O0-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s6, v16, 8
-; GFX9-O0-NEXT:    v_readlane_b32 s7, v16, 9
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    v_readlane_b32 s6, v16, 10
+; GFX9-O0-NEXT:    v_readlane_b32 s7, v16, 11
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 63
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(16)
 ; GFX9-O0-NEXT:    v_lshrrev_b64 v[29:30], s4, v[2:3]
@@ -847,72 +853,72 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v3
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v2
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v1
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v0
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v15
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v14
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v13
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 4
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 5
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 6
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 7
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 8
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 9
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 10
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 11
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execnz .LBB0_6
 ; GFX9-O0-NEXT:    s_branch .LBB0_1
 ; GFX9-O0-NEXT:  .LBB0_7: ; %udiv-preheader
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
@@ -993,51 +999,51 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v17
 ; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v15, s9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v14, s8
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v13, s7
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s4, 8
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s5, 9
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s4, 10
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s5, 11
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_6
 ; GFX9-O0-NEXT:  .LBB0_8: ; %udiv-bb1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -1074,14 +1080,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v10
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 0x7f
 ; GFX9-O0-NEXT:    v_sub_u32_e64 v3, s4, v4
 ; GFX9-O0-NEXT:    v_lshlrev_b64 v[5:6], v3, v[11:12]
@@ -1127,12 +1133,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v3
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v10
 ; GFX9-O0-NEXT:    v_or_b32_e64 v3, v3, v4
@@ -1147,39 +1153,39 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v4, s9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, s7
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-O0-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; GFX9-O0-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 6
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 7
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 8
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 9
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execz .LBB0_5
 ; GFX9-O0-NEXT:    s_branch .LBB0_7
 ; GFX9-O0-NEXT:  .LBB0_9: ; %udiv-end
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
@@ -1224,11 +1230,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX9-O0-NEXT:    ; kill: killed $vgpr4
 ; GFX9-O0-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 16a03badcb1329adedbccdab9c9561a4054bb8d2..e04cd7112560811ba55fe116ac6942173893aeb0 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -6,185 +6,187 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-LABEL: v_sdiv_v2i128_vv:
 ; SDAG:       ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    v_ashrrev_i32_e32 v24, 31, v3
-; SDAG-NEXT:    v_ashrrev_i32_e32 v25, 31, v11
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
+; SDAG-NEXT:    v_ashrrev_i32_e32 v26, 31, v3
+; SDAG-NEXT:    v_ashrrev_i32_e32 v27, 31, v11
+; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, 0, v0
+; SDAG-NEXT:    v_mov_b32_e32 v19, 0
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0x7f
-; SDAG-NEXT:    v_mov_b32_e32 v26, v24
-; SDAG-NEXT:    v_mov_b32_e32 v27, v25
-; SDAG-NEXT:    v_xor_b32_e32 v17, v24, v3
-; SDAG-NEXT:    v_xor_b32_e32 v18, v24, v2
-; SDAG-NEXT:    v_xor_b32_e32 v1, v24, v1
-; SDAG-NEXT:    v_xor_b32_e32 v0, v24, v0
-; SDAG-NEXT:    v_xor_b32_e32 v19, v25, v11
-; SDAG-NEXT:    v_xor_b32_e32 v20, v25, v10
-; SDAG-NEXT:    v_xor_b32_e32 v9, v25, v9
-; SDAG-NEXT:    v_xor_b32_e32 v8, v25, v8
-; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v0, v24
-; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v1, v24, vcc
-; SDAG-NEXT:    v_ffbh_u32_e32 v0, v2
-; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v18, v24, vcc
-; SDAG-NEXT:    v_add_i32_e64 v1, s[4:5], 32, v0
-; SDAG-NEXT:    v_ffbh_u32_e32 v18, v3
-; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, v17, v24, vcc
-; SDAG-NEXT:    v_or_b32_e32 v0, v2, v10
-; SDAG-NEXT:    v_ffbh_u32_e32 v17, v10
-; SDAG-NEXT:    v_min_u32_e32 v18, v1, v18
-; SDAG-NEXT:    v_sub_i32_e32 v28, vcc, v8, v25
-; SDAG-NEXT:    v_or_b32_e32 v1, v3, v11
-; SDAG-NEXT:    v_add_i32_e64 v8, s[4:5], 32, v17
-; SDAG-NEXT:    v_ffbh_u32_e32 v17, v11
-; SDAG-NEXT:    v_add_i32_e64 v18, s[4:5], 64, v18
-; SDAG-NEXT:    v_addc_u32_e64 v21, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v29, vcc, v9, v25, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; SDAG-NEXT:    v_ffbh_u32_e32 v1, v28
-; SDAG-NEXT:    v_min_u32_e32 v8, v8, v17
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[10:11]
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v21, 0, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v20, v25, vcc
-; SDAG-NEXT:    v_add_i32_e64 v9, s[8:9], 32, v1
-; SDAG-NEXT:    v_ffbh_u32_e32 v20, v29
-; SDAG-NEXT:    v_cndmask_b32_e64 v18, v18, v8, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v19, v25, vcc
-; SDAG-NEXT:    v_or_b32_e32 v8, v28, v0
-; SDAG-NEXT:    v_ffbh_u32_e32 v19, v0
-; SDAG-NEXT:    v_min_u32_e32 v20, v9, v20
-; SDAG-NEXT:    v_or_b32_e32 v9, v29, v1
-; SDAG-NEXT:    v_add_i32_e32 v19, vcc, 32, v19
+; SDAG-NEXT:    v_mov_b32_e32 v28, v26
+; SDAG-NEXT:    v_mov_b32_e32 v29, v27
+; SDAG-NEXT:    v_subb_u32_e32 v17, vcc, 0, v1, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v18, vcc, 0, v2, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; SDAG-NEXT:    v_cndmask_b32_e64 v17, v1, v17, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v16, v0, v16, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, 0, v3, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v1, v16
+; SDAG-NEXT:    v_ffbh_u32_e32 v18, v17
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v3, v0, s[4:5]
+; SDAG-NEXT:    v_sub_i32_e32 v20, vcc, 0, v8
+; SDAG-NEXT:    v_or_b32_e32 v0, v16, v2
+; SDAG-NEXT:    v_ffbh_u32_e32 v21, v2
+; SDAG-NEXT:    v_add_i32_e64 v22, s[4:5], 32, v1
+; SDAG-NEXT:    v_subb_u32_e32 v23, vcc, 0, v9, vcc
+; SDAG-NEXT:    v_or_b32_e32 v1, v17, v3
+; SDAG-NEXT:    v_add_i32_e64 v21, s[4:5], 32, v21
+; SDAG-NEXT:    v_min_u32_e32 v18, v22, v18
+; SDAG-NEXT:    v_ffbh_u32_e32 v22, v3
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[10:11]
+; SDAG-NEXT:    v_cndmask_b32_e64 v30, v9, v23, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, 0, v10, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v31, v8, v20, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[0:1]
+; SDAG-NEXT:    v_min_u32_e32 v1, v21, v22
+; SDAG-NEXT:    v_add_i32_e64 v8, s[8:9], 64, v18
+; SDAG-NEXT:    v_addc_u32_e64 v18, s[8:9], 0, 0, s[8:9]
+; SDAG-NEXT:    v_subb_u32_e32 v20, vcc, 0, v11, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, v10, v9, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v9, v31
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, v18, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v18, v8, v1, vcc
+; SDAG-NEXT:    v_ffbh_u32_e32 v21, v30
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v11, v20, s[4:5]
+; SDAG-NEXT:    v_or_b32_e32 v8, v31, v0
+; SDAG-NEXT:    v_ffbh_u32_e32 v11, v0
+; SDAG-NEXT:    v_add_i32_e32 v20, vcc, 32, v9
+; SDAG-NEXT:    v_or_b32_e32 v9, v30, v1
+; SDAG-NEXT:    v_add_i32_e32 v11, vcc, 32, v11
+; SDAG-NEXT:    v_min_u32_e32 v20, v20, v21
 ; SDAG-NEXT:    v_ffbh_u32_e32 v21, v1
-; SDAG-NEXT:    v_add_i32_e32 v20, vcc, 64, v20
-; SDAG-NEXT:    v_addc_u32_e64 v22, s[6:7], 0, 0, vcc
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT:    v_min_u32_e32 v8, v19, v21
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[0:1]
-; SDAG-NEXT:    v_cndmask_b32_e64 v9, v22, 0, s[6:7]
-; SDAG-NEXT:    s_or_b64 s[8:9], vcc, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, v20, v8, s[6:7]
+; SDAG-NEXT:    v_min_u32_e32 v8, v11, v21
+; SDAG-NEXT:    v_add_i32_e64 v9, s[4:5], 64, v20
+; SDAG-NEXT:    v_addc_u32_e64 v11, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, v11, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
 ; SDAG-NEXT:    v_sub_i32_e32 v8, vcc, v8, v18
-; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, v9, v17, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v17, 0x7f, v8
-; SDAG-NEXT:    v_subbrev_u32_e32 v18, vcc, 0, v16, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, v11, v10, vcc
+; SDAG-NEXT:    v_xor_b32_e32 v10, 0x7f, v8
+; SDAG-NEXT:    v_subbrev_u32_e32 v18, vcc, 0, v19, vcc
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[8:9]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s[4:5]
-; SDAG-NEXT:    v_subbrev_u32_e32 v19, vcc, 0, v16, vcc
-; SDAG-NEXT:    v_or_b32_e32 v16, v17, v18
+; SDAG-NEXT:    v_subbrev_u32_e32 v19, vcc, 0, v19, vcc
+; SDAG-NEXT:    v_or_b32_e32 v10, v10, v18
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v9, v19
+; SDAG-NEXT:    v_or_b32_e32 v11, v9, v19
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[18:19]
 ; SDAG-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[16:17]
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v20
-; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
-; SDAG-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v20, v11, 0, s[4:5]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; SDAG-NEXT:    v_and_b32_e32 v10, 1, v20
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v10
+; SDAG-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v20, v3, 0, s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v10, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v21, v3, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v16, v2, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v21, v2, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v22, v17, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v23, v16, 0, s[4:5]
 ; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_6
 ; SDAG-NEXT:  ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT:    v_add_i32_e32 v30, vcc, 1, v8
+; SDAG-NEXT:    v_add_i32_e32 v32, vcc, 1, v8
 ; SDAG-NEXT:    v_sub_i32_e64 v20, s[4:5], 63, v8
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
-; SDAG-NEXT:    v_addc_u32_e32 v31, vcc, 0, v9, vcc
-; SDAG-NEXT:    v_lshl_b64 v[20:21], v[2:3], v20
-; SDAG-NEXT:    v_addc_u32_e32 v32, vcc, 0, v18, vcc
-; SDAG-NEXT:    v_addc_u32_e32 v33, vcc, 0, v19, vcc
-; SDAG-NEXT:    v_or_b32_e32 v18, v30, v32
-; SDAG-NEXT:    v_sub_i32_e32 v34, vcc, 0x7f, v8
-; SDAG-NEXT:    v_or_b32_e32 v19, v31, v33
-; SDAG-NEXT:    v_lshl_b64 v[8:9], v[10:11], v34
-; SDAG-NEXT:    v_sub_i32_e32 v35, vcc, 64, v34
-; SDAG-NEXT:    v_lshl_b64 v[22:23], v[2:3], v34
+; SDAG-NEXT:    v_mov_b32_e32 v10, 0
+; SDAG-NEXT:    v_mov_b32_e32 v11, 0
+; SDAG-NEXT:    v_addc_u32_e32 v33, vcc, 0, v9, vcc
+; SDAG-NEXT:    v_lshl_b64 v[20:21], v[16:17], v20
+; SDAG-NEXT:    v_addc_u32_e32 v34, vcc, 0, v18, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, 0, v19, vcc
+; SDAG-NEXT:    v_or_b32_e32 v18, v32, v34
+; SDAG-NEXT:    v_sub_i32_e32 v24, vcc, 0x7f, v8
+; SDAG-NEXT:    v_or_b32_e32 v19, v33, v35
+; SDAG-NEXT:    v_lshl_b64 v[8:9], v[2:3], v24
+; SDAG-NEXT:    v_sub_i32_e32 v25, vcc, 64, v24
+; SDAG-NEXT:    v_lshl_b64 v[22:23], v[16:17], v24
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; SDAG-NEXT:    v_lshr_b64 v[18:19], v[2:3], v35
+; SDAG-NEXT:    v_lshr_b64 v[18:19], v[16:17], v25
 ; SDAG-NEXT:    v_or_b32_e32 v9, v9, v19
 ; SDAG-NEXT:    v_or_b32_e32 v8, v8, v18
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v34
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v24
 ; SDAG-NEXT:    v_cndmask_b32_e64 v9, v21, v9, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v8, v20, v8, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v21, 0, v23, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v20, 0, v22, s[4:5]
-; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v34
-; SDAG-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v24
+; SDAG-NEXT:    v_cndmask_b32_e64 v9, v9, v3, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, v8, v2, s[4:5]
 ; SDAG-NEXT:    v_mov_b32_e32 v18, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v19, 0
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_5
 ; SDAG-NEXT:  ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT:    v_lshr_b64 v[16:17], v[2:3], v30
-; SDAG-NEXT:    v_sub_i32_e32 v35, vcc, 64, v30
-; SDAG-NEXT:    v_subrev_i32_e32 v36, vcc, 64, v30
-; SDAG-NEXT:    v_lshr_b64 v[37:38], v[10:11], v30
-; SDAG-NEXT:    v_add_i32_e32 v34, vcc, -1, v28
+; SDAG-NEXT:    v_lshr_b64 v[10:11], v[16:17], v32
+; SDAG-NEXT:    v_sub_i32_e32 v37, vcc, 64, v32
+; SDAG-NEXT:    v_subrev_i32_e32 v48, vcc, 64, v32
+; SDAG-NEXT:    v_lshr_b64 v[24:25], v[2:3], v32
+; SDAG-NEXT:    v_add_i32_e32 v36, vcc, -1, v31
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0
 ; SDAG-NEXT:    v_mov_b32_e32 v22, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v23, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v18, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v19, 0
-; SDAG-NEXT:    v_lshl_b64 v[48:49], v[10:11], v35
-; SDAG-NEXT:    v_lshr_b64 v[10:11], v[10:11], v36
-; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, -1, v29, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v17, v49
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v48
-; SDAG-NEXT:    v_addc_u32_e32 v36, vcc, -1, v0, vcc
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v30
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v11, v17, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v16, v10, v16, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v38, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, v37, s[4:5]
-; SDAG-NEXT:    v_addc_u32_e32 v37, vcc, -1, v1, vcc
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v30
-; SDAG-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
+; SDAG-NEXT:    v_lshl_b64 v[38:39], v[2:3], v37
+; SDAG-NEXT:    v_lshr_b64 v[2:3], v[2:3], v48
+; SDAG-NEXT:    v_addc_u32_e32 v37, vcc, -1, v30, vcc
+; SDAG-NEXT:    v_or_b32_e32 v11, v11, v39
+; SDAG-NEXT:    v_or_b32_e32 v10, v10, v38
+; SDAG-NEXT:    v_addc_u32_e32 v38, vcc, -1, v0, vcc
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v32
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v25, 0, v25, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v24, 0, v24, s[4:5]
+; SDAG-NEXT:    v_addc_u32_e32 v39, vcc, -1, v1, vcc
+; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v32
+; SDAG-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v2, v2, v16, vcc
+; SDAG-NEXT:    v_mov_b32_e32 v11, 0
 ; SDAG-NEXT:  .LBB0_3: ; %udiv-do-while3
 ; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v16, 31, v3
+; SDAG-NEXT:    v_lshl_b64 v[16:17], v[24:25], 1
+; SDAG-NEXT:    v_lshrrev_b32_e32 v10, 31, v3
 ; SDAG-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v38, 31, v9
+; SDAG-NEXT:    v_lshrrev_b32_e32 v24, 31, v9
 ; SDAG-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v39, 31, v21
+; SDAG-NEXT:    v_lshrrev_b32_e32 v25, 31, v21
 ; SDAG-NEXT:    v_lshl_b64 v[20:21], v[20:21], 1
-; SDAG-NEXT:    v_or_b32_e32 v10, v10, v16
-; SDAG-NEXT:    v_or_b32_e32 v2, v2, v38
-; SDAG-NEXT:    v_or_b32_e32 v8, v8, v39
+; SDAG-NEXT:    v_or_b32_e32 v16, v16, v10
+; SDAG-NEXT:    v_or_b32_e32 v2, v2, v24
+; SDAG-NEXT:    v_or_b32_e32 v8, v8, v25
 ; SDAG-NEXT:    v_or_b32_e32 v9, v19, v9
-; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, v34, v2
+; SDAG-NEXT:    v_sub_i32_e32 v10, vcc, v36, v2
 ; SDAG-NEXT:    v_or_b32_e32 v8, v18, v8
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v35, v3, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v36, v10, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v37, v11, vcc
-; SDAG-NEXT:    v_ashrrev_i32_e32 v38, 31, v16
-; SDAG-NEXT:    v_and_b32_e32 v39, v38, v28
-; SDAG-NEXT:    v_and_b32_e32 v48, v38, v29
-; SDAG-NEXT:    v_and_b32_e32 v49, v38, v0
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v38
-; SDAG-NEXT:    v_and_b32_e32 v38, v38, v1
-; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v2, v39
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v37, v3, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v38, v16, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v39, v17, vcc
+; SDAG-NEXT:    v_ashrrev_i32_e32 v24, 31, v10
+; SDAG-NEXT:    v_and_b32_e32 v25, v24, v31
+; SDAG-NEXT:    v_and_b32_e32 v48, v24, v30
+; SDAG-NEXT:    v_and_b32_e32 v49, v24, v0
+; SDAG-NEXT:    v_and_b32_e32 v10, 1, v24
+; SDAG-NEXT:    v_and_b32_e32 v50, v24, v1
+; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v2, v25
 ; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v48, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v10, v49, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, v11, v38, vcc
-; SDAG-NEXT:    v_add_i32_e32 v30, vcc, -1, v30
-; SDAG-NEXT:    v_addc_u32_e32 v31, vcc, -1, v31, vcc
-; SDAG-NEXT:    v_addc_u32_e32 v32, vcc, -1, v32, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v24, vcc, v16, v49, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v25, vcc, v17, v50, vcc
+; SDAG-NEXT:    v_add_i32_e32 v32, vcc, -1, v32
 ; SDAG-NEXT:    v_addc_u32_e32 v33, vcc, -1, v33, vcc
-; SDAG-NEXT:    v_or_b32_e32 v38, v30, v32
-; SDAG-NEXT:    v_or_b32_e32 v39, v31, v33
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[38:39]
+; SDAG-NEXT:    v_addc_u32_e32 v34, vcc, -1, v34, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, -1, v35, vcc
+; SDAG-NEXT:    v_or_b32_e32 v16, v32, v34
+; SDAG-NEXT:    v_or_b32_e32 v17, v33, v35
+; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[16:17]
 ; SDAG-NEXT:    v_or_b32_e32 v21, v23, v21
 ; SDAG-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
 ; SDAG-NEXT:    v_or_b32_e32 v20, v22, v20
-; SDAG-NEXT:    v_mov_b32_e32 v23, v17
-; SDAG-NEXT:    v_mov_b32_e32 v22, v16
+; SDAG-NEXT:    v_mov_b32_e32 v23, v11
+; SDAG-NEXT:    v_mov_b32_e32 v22, v10
 ; SDAG-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; SDAG-NEXT:    s_cbranch_execnz .LBB0_3
 ; SDAG-NEXT:  ; %bb.4: ; %Flow13
@@ -196,68 +198,70 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_lshl_b64 v[2:3], v[20:21], 1
 ; SDAG-NEXT:    v_or_b32_e32 v0, v0, v8
 ; SDAG-NEXT:    v_or_b32_e32 v20, v19, v1
-; SDAG-NEXT:    v_or_b32_e32 v21, v17, v3
-; SDAG-NEXT:    v_or_b32_e32 v17, v18, v0
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v2
+; SDAG-NEXT:    v_or_b32_e32 v22, v11, v3
+; SDAG-NEXT:    v_or_b32_e32 v21, v18, v0
+; SDAG-NEXT:    v_or_b32_e32 v23, v10, v2
 ; SDAG-NEXT:  .LBB0_6: ; %Flow16
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT:    v_ashrrev_i32_e32 v18, 31, v7
-; SDAG-NEXT:    v_ashrrev_i32_e32 v19, 31, v15
+; SDAG-NEXT:    v_ashrrev_i32_e32 v16, 31, v7
+; SDAG-NEXT:    v_ashrrev_i32_e32 v17, 31, v15
+; SDAG-NEXT:    v_sub_i32_e32 v0, vcc, 0, v4
 ; SDAG-NEXT:    v_mov_b32_e32 v9, 0
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0x7f
-; SDAG-NEXT:    v_mov_b32_e32 v22, v18
-; SDAG-NEXT:    v_mov_b32_e32 v23, v19
-; SDAG-NEXT:    v_xor_b32_e32 v0, v18, v7
-; SDAG-NEXT:    v_xor_b32_e32 v1, v18, v6
-; SDAG-NEXT:    v_xor_b32_e32 v3, v18, v5
-; SDAG-NEXT:    v_xor_b32_e32 v2, v18, v4
-; SDAG-NEXT:    v_xor_b32_e32 v6, v19, v15
-; SDAG-NEXT:    v_xor_b32_e32 v7, v19, v14
-; SDAG-NEXT:    v_xor_b32_e32 v8, v19, v13
-; SDAG-NEXT:    v_xor_b32_e32 v10, v19, v12
-; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v2, v18
-; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v18, vcc
-; SDAG-NEXT:    v_ffbh_u32_e32 v5, v2
-; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v1, v18, vcc
-; SDAG-NEXT:    v_add_i32_e64 v1, s[4:5], 32, v5
-; SDAG-NEXT:    v_ffbh_u32_e32 v11, v3
-; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v0, v18, vcc
+; SDAG-NEXT:    v_mov_b32_e32 v18, v16
+; SDAG-NEXT:    v_mov_b32_e32 v19, v17
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, 0, v5, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v8, vcc, 0, v6, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v5, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v4, v0, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, 0, v7, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v1, v2
+; SDAG-NEXT:    v_ffbh_u32_e32 v6, v3
+; SDAG-NEXT:    v_cndmask_b32_e64 v5, v7, v0, s[4:5]
+; SDAG-NEXT:    v_sub_i32_e32 v7, vcc, 0, v12
 ; SDAG-NEXT:    v_or_b32_e32 v0, v2, v4
-; SDAG-NEXT:    v_ffbh_u32_e32 v12, v4
-; SDAG-NEXT:    v_min_u32_e32 v11, v1, v11
-; SDAG-NEXT:    v_sub_i32_e32 v28, vcc, v10, v19
+; SDAG-NEXT:    v_ffbh_u32_e32 v8, v4
+; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], 32, v1
+; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, 0, v13, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v1, v3, v5
-; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], 32, v12
-; SDAG-NEXT:    v_ffbh_u32_e32 v12, v5
-; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], 64, v11
-; SDAG-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v29, vcc, v8, v19, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; SDAG-NEXT:    v_ffbh_u32_e32 v1, v28
-; SDAG-NEXT:    v_min_u32_e32 v8, v10, v12
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, v13, 0, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v7, v19, vcc
-; SDAG-NEXT:    v_add_i32_e64 v7, s[8:9], 32, v1
-; SDAG-NEXT:    v_ffbh_u32_e32 v12, v29
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, v11, v8, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v6, v19, vcc
-; SDAG-NEXT:    v_or_b32_e32 v6, v28, v0
-; SDAG-NEXT:    v_ffbh_u32_e32 v11, v0
-; SDAG-NEXT:    v_min_u32_e32 v12, v7, v12
-; SDAG-NEXT:    v_or_b32_e32 v7, v29, v1
-; SDAG-NEXT:    v_add_i32_e32 v11, vcc, 32, v11
-; SDAG-NEXT:    v_ffbh_u32_e32 v13, v1
-; SDAG-NEXT:    v_add_i32_e32 v12, vcc, 64, v12
-; SDAG-NEXT:    v_addc_u32_e64 v14, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_add_i32_e64 v8, s[4:5], 32, v8
+; SDAG-NEXT:    v_ffbh_u32_e32 v30, v5
+; SDAG-NEXT:    v_min_u32_e32 v6, v10, v6
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, 0, v14, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[14:15]
+; SDAG-NEXT:    v_cndmask_b32_e64 v24, v13, v11, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v25, v12, v7, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[0:1]
+; SDAG-NEXT:    v_min_u32_e32 v1, v8, v30
+; SDAG-NEXT:    v_add_i32_e64 v6, s[8:9], 64, v6
+; SDAG-NEXT:    v_addc_u32_e64 v7, s[8:9], 0, 0, s[8:9]
+; SDAG-NEXT:    v_subb_u32_e32 v8, vcc, 0, v15, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, v14, v10, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v10, v25
+; SDAG-NEXT:    v_ffbh_u32_e32 v11, v24
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, v7, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v13, v6, v1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v15, v8, s[4:5]
+; SDAG-NEXT:    v_or_b32_e32 v6, v25, v0
+; SDAG-NEXT:    v_ffbh_u32_e32 v8, v0
+; SDAG-NEXT:    v_add_i32_e32 v10, vcc, 32, v10
+; SDAG-NEXT:    v_or_b32_e32 v7, v24, v1
+; SDAG-NEXT:    v_add_i32_e32 v8, vcc, 32, v8
+; SDAG-NEXT:    v_ffbh_u32_e32 v14, v1
+; SDAG-NEXT:    v_min_u32_e32 v10, v10, v11
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT:    v_min_u32_e32 v6, v11, v13
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[0:1]
-; SDAG-NEXT:    v_cndmask_b32_e64 v7, v14, 0, s[6:7]
-; SDAG-NEXT:    s_or_b64 s[8:9], vcc, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v6, v12, v6, s[6:7]
-; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v6, v8
-; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v7, v10, vcc
+; SDAG-NEXT:    v_min_u32_e32 v6, v8, v14
+; SDAG-NEXT:    v_add_i32_e64 v7, s[4:5], 64, v10
+; SDAG-NEXT:    v_addc_u32_e64 v8, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, v8, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
+; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v6, v13
+; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v8, v12, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v10, 0x7f, v6
 ; SDAG-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v9, vcc
 ; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[6:7]
@@ -272,7 +276,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
 ; SDAG-NEXT:    v_and_b32_e32 v10, 1, v12
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v10
-; SDAG-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v13, v5, 0, s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
 ; SDAG-NEXT:    v_cndmask_b32_e64 v11, v4, 0, s[4:5]
@@ -318,7 +322,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_sub_i32_e32 v35, vcc, 64, v30
 ; SDAG-NEXT:    v_subrev_i32_e32 v36, vcc, 64, v30
 ; SDAG-NEXT:    v_lshr_b64 v[37:38], v[4:5], v30
-; SDAG-NEXT:    v_add_i32_e32 v34, vcc, -1, v28
+; SDAG-NEXT:    v_add_i32_e32 v34, vcc, -1, v25
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0
 ; SDAG-NEXT:    v_mov_b32_e32 v14, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v15, 0
@@ -326,7 +330,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_mov_b32_e32 v13, 0
 ; SDAG-NEXT:    v_lshl_b64 v[48:49], v[4:5], v35
 ; SDAG-NEXT:    v_lshr_b64 v[4:5], v[4:5], v36
-; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, -1, v29, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, -1, v24, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v11, v11, v49
 ; SDAG-NEXT:    v_or_b32_e32 v10, v10, v48
 ; SDAG-NEXT:    v_addc_u32_e32 v36, vcc, -1, v0, vcc
@@ -363,8 +367,8 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_and_b32_e32 v10, 1, v15
 ; SDAG-NEXT:    v_and_b32_e32 v38, v15, v1
 ; SDAG-NEXT:    v_and_b32_e32 v39, v15, v0
-; SDAG-NEXT:    v_and_b32_e32 v48, v15, v29
-; SDAG-NEXT:    v_and_b32_e32 v15, v15, v28
+; SDAG-NEXT:    v_and_b32_e32 v48, v15, v24
+; SDAG-NEXT:    v_and_b32_e32 v15, v15, v25
 ; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v2, v15
 ; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v48, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v4, v39, vcc
@@ -396,14 +400,14 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_or_b32_e32 v10, v10, v2
 ; SDAG-NEXT:  .LBB0_12: ; %Flow12
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT:    v_xor_b32_e32 v3, v27, v26
-; SDAG-NEXT:    v_xor_b32_e32 v2, v25, v24
-; SDAG-NEXT:    v_xor_b32_e32 v7, v23, v22
-; SDAG-NEXT:    v_xor_b32_e32 v6, v19, v18
+; SDAG-NEXT:    v_xor_b32_e32 v3, v29, v28
+; SDAG-NEXT:    v_xor_b32_e32 v2, v27, v26
+; SDAG-NEXT:    v_xor_b32_e32 v7, v19, v18
+; SDAG-NEXT:    v_xor_b32_e32 v6, v17, v16
 ; SDAG-NEXT:    v_xor_b32_e32 v4, v20, v3
-; SDAG-NEXT:    v_xor_b32_e32 v5, v17, v2
-; SDAG-NEXT:    v_xor_b32_e32 v1, v21, v3
-; SDAG-NEXT:    v_xor_b32_e32 v0, v16, v2
+; SDAG-NEXT:    v_xor_b32_e32 v5, v21, v2
+; SDAG-NEXT:    v_xor_b32_e32 v1, v22, v3
+; SDAG-NEXT:    v_xor_b32_e32 v0, v23, v2
 ; SDAG-NEXT:    v_xor_b32_e32 v8, v13, v7
 ; SDAG-NEXT:    v_xor_b32_e32 v9, v11, v6
 ; SDAG-NEXT:    v_xor_b32_e32 v11, v14, v7
@@ -1553,118 +1557,119 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v28, 31, v3
-; SDAG-NEXT:    v_ashrrev_i32_e32 v16, 31, v11
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
+; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, 0, v0
+; SDAG-NEXT:    v_mov_b32_e32 v19, 0
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0x7f
 ; SDAG-NEXT:    v_mov_b32_e32 v29, v28
-; SDAG-NEXT:    v_xor_b32_e32 v18, v3, v28
-; SDAG-NEXT:    v_xor_b32_e32 v19, v2, v28
-; SDAG-NEXT:    v_xor_b32_e32 v1, v1, v28
-; SDAG-NEXT:    v_xor_b32_e32 v0, v0, v28
-; SDAG-NEXT:    v_xor_b32_e32 v11, v11, v16
-; SDAG-NEXT:    v_xor_b32_e32 v10, v10, v16
-; SDAG-NEXT:    v_xor_b32_e32 v20, v9, v16
-; SDAG-NEXT:    v_xor_b32_e32 v9, v8, v16
-; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v0, v28
-; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v1, v28, vcc
-; SDAG-NEXT:    v_ffbh_u32_e32 v1, v2
-; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v19, v28, vcc
-; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], 32, v1
-; SDAG-NEXT:    v_ffbh_u32_e32 v21, v3
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v18, v28, vcc
-; SDAG-NEXT:    v_or_b32_e32 v8, v2, v0
-; SDAG-NEXT:    v_ffbh_u32_e32 v18, v0
-; SDAG-NEXT:    v_min_u32_e32 v19, v19, v21
-; SDAG-NEXT:    v_sub_i32_e32 v31, vcc, v9, v16
-; SDAG-NEXT:    v_or_b32_e32 v9, v3, v1
+; SDAG-NEXT:    v_subb_u32_e32 v17, vcc, 0, v1, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v18, vcc, 0, v2, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; SDAG-NEXT:    v_cndmask_b32_e64 v17, v1, v17, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v16, v0, v16, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, 0, v3, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, v2, v18, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v18, v16
+; SDAG-NEXT:    v_ffbh_u32_e32 v20, v17
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; SDAG-NEXT:    v_sub_i32_e32 v21, vcc, 0, v8
+; SDAG-NEXT:    v_or_b32_e32 v2, v16, v0
+; SDAG-NEXT:    v_ffbh_u32_e32 v22, v0
 ; SDAG-NEXT:    v_add_i32_e64 v18, s[4:5], 32, v18
-; SDAG-NEXT:    v_ffbh_u32_e32 v21, v1
-; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], 64, v19
-; SDAG-NEXT:    v_addc_u32_e64 v22, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v30, vcc, v20, v16, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
-; SDAG-NEXT:    v_ffbh_u32_e32 v9, v31
-; SDAG-NEXT:    v_min_u32_e32 v18, v18, v21
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[0:1]
-; SDAG-NEXT:    v_cndmask_b32_e64 v20, v22, 0, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v8, vcc, v10, v16, vcc
-; SDAG-NEXT:    v_add_i32_e64 v21, s[8:9], 32, v9
-; SDAG-NEXT:    v_ffbh_u32_e32 v22, v30
-; SDAG-NEXT:    v_cndmask_b32_e64 v18, v19, v18, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, v11, v16, vcc
-; SDAG-NEXT:    v_or_b32_e32 v10, v31, v8
-; SDAG-NEXT:    v_ffbh_u32_e32 v16, v8
-; SDAG-NEXT:    v_min_u32_e32 v19, v21, v22
-; SDAG-NEXT:    v_or_b32_e32 v11, v30, v9
-; SDAG-NEXT:    v_add_i32_e32 v16, vcc, 32, v16
-; SDAG-NEXT:    v_ffbh_u32_e32 v21, v9
-; SDAG-NEXT:    v_add_i32_e32 v19, vcc, 64, v19
-; SDAG-NEXT:    v_addc_u32_e64 v22, s[6:7], 0, 0, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; SDAG-NEXT:    v_min_u32_e32 v10, v16, v21
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[8:9]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, v22, 0, s[6:7]
-; SDAG-NEXT:    s_or_b64 s[8:9], vcc, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, v19, v10, s[6:7]
-; SDAG-NEXT:    v_sub_i32_e32 v10, vcc, v10, v18
-; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, v11, v20, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v16, 0x7f, v10
-; SDAG-NEXT:    v_subbrev_u32_e32 v18, vcc, 0, v17, vcc
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[10:11]
+; SDAG-NEXT:    v_subb_u32_e32 v23, vcc, 0, v9, vcc
+; SDAG-NEXT:    v_or_b32_e32 v3, v17, v1
+; SDAG-NEXT:    v_add_i32_e64 v22, s[4:5], 32, v22
+; SDAG-NEXT:    v_ffbh_u32_e32 v24, v1
+; SDAG-NEXT:    v_min_u32_e32 v18, v18, v20
+; SDAG-NEXT:    v_subb_u32_e32 v20, vcc, 0, v10, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[10:11]
+; SDAG-NEXT:    v_cndmask_b32_e64 v30, v9, v23, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v31, v8, v21, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[2:3]
+; SDAG-NEXT:    v_min_u32_e32 v3, v22, v24
+; SDAG-NEXT:    v_add_i32_e64 v8, s[8:9], 64, v18
+; SDAG-NEXT:    v_addc_u32_e64 v9, s[8:9], 0, 0, s[8:9]
+; SDAG-NEXT:    v_subb_u32_e32 v18, vcc, 0, v11, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v10, v20, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v10, v31
+; SDAG-NEXT:    v_ffbh_u32_e32 v20, v30
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT:    v_cndmask_b32_e64 v21, v9, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v22, v8, v3, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v11, v18, s[4:5]
+; SDAG-NEXT:    v_or_b32_e32 v8, v31, v2
+; SDAG-NEXT:    v_ffbh_u32_e32 v11, v2
+; SDAG-NEXT:    v_add_i32_e32 v10, vcc, 32, v10
+; SDAG-NEXT:    v_or_b32_e32 v9, v30, v3
+; SDAG-NEXT:    v_add_i32_e32 v11, vcc, 32, v11
+; SDAG-NEXT:    v_ffbh_u32_e32 v18, v3
+; SDAG-NEXT:    v_min_u32_e32 v10, v10, v20
+; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; SDAG-NEXT:    v_min_u32_e32 v8, v11, v18
+; SDAG-NEXT:    v_add_i32_e64 v9, s[4:5], 64, v10
+; SDAG-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, v10, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; SDAG-NEXT:    v_sub_i32_e32 v8, vcc, v8, v22
+; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, v10, v21, vcc
+; SDAG-NEXT:    v_xor_b32_e32 v10, 0x7f, v8
+; SDAG-NEXT:    v_subbrev_u32_e32 v18, vcc, 0, v19, vcc
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[8:9]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s[4:5]
-; SDAG-NEXT:    v_subbrev_u32_e32 v19, vcc, 0, v17, vcc
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v18
+; SDAG-NEXT:    v_subbrev_u32_e32 v19, vcc, 0, v19, vcc
+; SDAG-NEXT:    v_or_b32_e32 v10, v10, v18
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v11, v19
+; SDAG-NEXT:    v_or_b32_e32 v11, v9, v19
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[18:19]
 ; SDAG-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[16:17]
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v20
-; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
-; SDAG-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; SDAG-NEXT:    v_and_b32_e32 v10, 1, v20
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v10
+; SDAG-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v35, v1, 0, s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
 ; SDAG-NEXT:    v_cndmask_b32_e64 v32, v0, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v27, v3, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v33, v2, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v27, v17, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v33, v16, 0, s[4:5]
 ; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_6
 ; SDAG-NEXT:  ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT:    v_add_i32_e32 v32, vcc, 1, v10
-; SDAG-NEXT:    v_sub_i32_e64 v20, s[4:5], 63, v10
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
-; SDAG-NEXT:    v_addc_u32_e32 v33, vcc, 0, v11, vcc
-; SDAG-NEXT:    v_lshl_b64 v[20:21], v[2:3], v20
+; SDAG-NEXT:    v_add_i32_e32 v32, vcc, 1, v8
+; SDAG-NEXT:    v_sub_i32_e64 v20, s[4:5], 63, v8
+; SDAG-NEXT:    v_mov_b32_e32 v10, 0
+; SDAG-NEXT:    v_mov_b32_e32 v11, 0
+; SDAG-NEXT:    v_addc_u32_e32 v33, vcc, 0, v9, vcc
+; SDAG-NEXT:    v_lshl_b64 v[20:21], v[16:17], v20
 ; SDAG-NEXT:    v_addc_u32_e32 v34, vcc, 0, v18, vcc
 ; SDAG-NEXT:    v_addc_u32_e32 v35, vcc, 0, v19, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v18, v32, v34
-; SDAG-NEXT:    v_sub_i32_e32 v24, vcc, 0x7f, v10
+; SDAG-NEXT:    v_sub_i32_e32 v24, vcc, 0x7f, v8
 ; SDAG-NEXT:    v_or_b32_e32 v19, v33, v35
-; SDAG-NEXT:    v_lshl_b64 v[10:11], v[0:1], v24
+; SDAG-NEXT:    v_lshl_b64 v[8:9], v[0:1], v24
 ; SDAG-NEXT:    v_sub_i32_e32 v25, vcc, 64, v24
-; SDAG-NEXT:    v_lshl_b64 v[22:23], v[2:3], v24
+; SDAG-NEXT:    v_lshl_b64 v[22:23], v[16:17], v24
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; SDAG-NEXT:    v_lshr_b64 v[18:19], v[2:3], v25
-; SDAG-NEXT:    v_or_b32_e32 v11, v11, v19
-; SDAG-NEXT:    v_or_b32_e32 v10, v10, v18
+; SDAG-NEXT:    v_lshr_b64 v[18:19], v[16:17], v25
+; SDAG-NEXT:    v_or_b32_e32 v9, v9, v19
+; SDAG-NEXT:    v_or_b32_e32 v8, v8, v18
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v24
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, v21, v11, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, v20, v10, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v9, v21, v9, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, v20, v8, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v21, 0, v23, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v20, 0, v22, s[4:5]
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v24
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, v11, v1, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, v10, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v9, v9, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, v8, v0, s[4:5]
 ; SDAG-NEXT:    v_mov_b32_e32 v18, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v19, 0
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_5
 ; SDAG-NEXT:  ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT:    v_lshr_b64 v[16:17], v[2:3], v32
+; SDAG-NEXT:    v_lshr_b64 v[10:11], v[16:17], v32
 ; SDAG-NEXT:    v_sub_i32_e32 v26, vcc, 64, v32
 ; SDAG-NEXT:    v_subrev_i32_e32 v37, vcc, 64, v32
 ; SDAG-NEXT:    v_lshr_b64 v[24:25], v[0:1], v32
@@ -1677,43 +1682,43 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_lshl_b64 v[26:27], v[0:1], v26
 ; SDAG-NEXT:    v_lshr_b64 v[48:49], v[0:1], v37
 ; SDAG-NEXT:    v_addc_u32_e32 v37, vcc, -1, v30, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v17, v27
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v26
-; SDAG-NEXT:    v_addc_u32_e32 v38, vcc, -1, v8, vcc
+; SDAG-NEXT:    v_or_b32_e32 v11, v11, v27
+; SDAG-NEXT:    v_or_b32_e32 v10, v10, v26
+; SDAG-NEXT:    v_addc_u32_e32 v38, vcc, -1, v2, vcc
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v32
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v49, v17, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v16, v48, v16, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, v49, v11, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, v48, v10, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v27, 0, v25, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v26, 0, v24, s[4:5]
-; SDAG-NEXT:    v_addc_u32_e32 v39, vcc, -1, v9, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v39, vcc, -1, v3, vcc
 ; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v32
-; SDAG-NEXT:    v_cndmask_b32_e32 v25, v17, v3, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v24, v16, v2, vcc
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
+; SDAG-NEXT:    v_cndmask_b32_e32 v25, v11, v17, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v24, v10, v16, vcc
+; SDAG-NEXT:    v_mov_b32_e32 v11, 0
 ; SDAG-NEXT:  .LBB2_3: ; %udiv-do-while3
 ; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SDAG-NEXT:    v_lshl_b64 v[26:27], v[26:27], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v16, 31, v25
+; SDAG-NEXT:    v_lshrrev_b32_e32 v10, 31, v25
 ; SDAG-NEXT:    v_lshl_b64 v[24:25], v[24:25], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v48, 31, v11
-; SDAG-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; SDAG-NEXT:    v_lshrrev_b32_e32 v48, 31, v9
+; SDAG-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
 ; SDAG-NEXT:    v_lshrrev_b32_e32 v49, 31, v21
 ; SDAG-NEXT:    v_lshl_b64 v[20:21], v[20:21], 1
-; SDAG-NEXT:    v_or_b32_e32 v26, v26, v16
+; SDAG-NEXT:    v_or_b32_e32 v26, v26, v10
 ; SDAG-NEXT:    v_or_b32_e32 v24, v24, v48
-; SDAG-NEXT:    v_or_b32_e32 v10, v10, v49
-; SDAG-NEXT:    v_or_b32_e32 v11, v19, v11
-; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, v36, v24
-; SDAG-NEXT:    v_or_b32_e32 v10, v18, v10
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v37, v25, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v38, v26, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v39, v27, vcc
-; SDAG-NEXT:    v_ashrrev_i32_e32 v16, 31, v16
-; SDAG-NEXT:    v_and_b32_e32 v48, v16, v31
-; SDAG-NEXT:    v_and_b32_e32 v49, v16, v30
-; SDAG-NEXT:    v_and_b32_e32 v50, v16, v8
-; SDAG-NEXT:    v_and_b32_e32 v51, v16, v9
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v16
+; SDAG-NEXT:    v_or_b32_e32 v8, v8, v49
+; SDAG-NEXT:    v_or_b32_e32 v9, v19, v9
+; SDAG-NEXT:    v_sub_i32_e32 v10, vcc, v36, v24
+; SDAG-NEXT:    v_or_b32_e32 v8, v18, v8
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v37, v25, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v38, v26, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v39, v27, vcc
+; SDAG-NEXT:    v_ashrrev_i32_e32 v10, 31, v10
+; SDAG-NEXT:    v_and_b32_e32 v48, v10, v31
+; SDAG-NEXT:    v_and_b32_e32 v49, v10, v30
+; SDAG-NEXT:    v_and_b32_e32 v50, v10, v2
+; SDAG-NEXT:    v_and_b32_e32 v51, v10, v3
+; SDAG-NEXT:    v_and_b32_e32 v10, 1, v10
 ; SDAG-NEXT:    v_sub_i32_e32 v24, vcc, v24, v48
 ; SDAG-NEXT:    v_subb_u32_e32 v25, vcc, v25, v49, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v26, vcc, v26, v50, vcc
@@ -1728,137 +1733,138 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_or_b32_e32 v21, v23, v21
 ; SDAG-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
 ; SDAG-NEXT:    v_or_b32_e32 v20, v22, v20
-; SDAG-NEXT:    v_mov_b32_e32 v23, v17
-; SDAG-NEXT:    v_mov_b32_e32 v22, v16
+; SDAG-NEXT:    v_mov_b32_e32 v23, v11
+; SDAG-NEXT:    v_mov_b32_e32 v22, v10
 ; SDAG-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; SDAG-NEXT:    s_cbranch_execnz .LBB2_3
 ; SDAG-NEXT:  ; %bb.4: ; %Flow13
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; SDAG-NEXT:  .LBB2_5: ; %Flow14
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; SDAG-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
 ; SDAG-NEXT:    v_lshrrev_b32_e32 v22, 31, v21
 ; SDAG-NEXT:    v_lshl_b64 v[20:21], v[20:21], 1
-; SDAG-NEXT:    v_or_b32_e32 v10, v10, v22
-; SDAG-NEXT:    v_or_b32_e32 v35, v19, v11
-; SDAG-NEXT:    v_or_b32_e32 v27, v17, v21
-; SDAG-NEXT:    v_or_b32_e32 v32, v18, v10
-; SDAG-NEXT:    v_or_b32_e32 v33, v16, v20
+; SDAG-NEXT:    v_or_b32_e32 v8, v8, v22
+; SDAG-NEXT:    v_or_b32_e32 v35, v19, v9
+; SDAG-NEXT:    v_or_b32_e32 v27, v11, v21
+; SDAG-NEXT:    v_or_b32_e32 v32, v18, v8
+; SDAG-NEXT:    v_or_b32_e32 v33, v10, v20
 ; SDAG-NEXT:  .LBB2_6: ; %Flow16
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v26, 31, v7
-; SDAG-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
+; SDAG-NEXT:    v_sub_i32_e32 v8, vcc, 0, v4
+; SDAG-NEXT:    v_mov_b32_e32 v18, 0
 ; SDAG-NEXT:    s_mov_b64 s[10:11], 0x7f
 ; SDAG-NEXT:    v_mov_b32_e32 v34, v26
-; SDAG-NEXT:    v_xor_b32_e32 v10, v7, v26
-; SDAG-NEXT:    v_xor_b32_e32 v11, v6, v26
-; SDAG-NEXT:    v_xor_b32_e32 v5, v5, v26
-; SDAG-NEXT:    v_xor_b32_e32 v4, v4, v26
-; SDAG-NEXT:    v_xor_b32_e32 v15, v15, v16
-; SDAG-NEXT:    v_xor_b32_e32 v14, v14, v16
-; SDAG-NEXT:    v_xor_b32_e32 v13, v13, v16
-; SDAG-NEXT:    v_xor_b32_e32 v12, v12, v16
-; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v4, v26
-; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v5, v26, vcc
-; SDAG-NEXT:    v_ffbh_u32_e32 v5, v6
-; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v11, v26, vcc
-; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], 32, v5
-; SDAG-NEXT:    v_ffbh_u32_e32 v18, v7
-; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v10, v26, vcc
-; SDAG-NEXT:    v_or_b32_e32 v10, v6, v4
-; SDAG-NEXT:    v_ffbh_u32_e32 v19, v4
-; SDAG-NEXT:    v_min_u32_e32 v18, v11, v18
-; SDAG-NEXT:    v_sub_i32_e32 v37, vcc, v12, v16
-; SDAG-NEXT:    v_or_b32_e32 v11, v7, v5
-; SDAG-NEXT:    v_add_i32_e64 v12, s[4:5], 32, v19
-; SDAG-NEXT:    v_ffbh_u32_e32 v19, v5
-; SDAG-NEXT:    v_add_i32_e64 v18, s[4:5], 64, v18
-; SDAG-NEXT:    v_addc_u32_e64 v20, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v36, vcc, v13, v16, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[10:11]
+; SDAG-NEXT:    v_subb_u32_e32 v9, vcc, 0, v5, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, 0, v6, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; SDAG-NEXT:    v_cndmask_b32_e64 v9, v5, v9, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, v4, v8, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, 0, v7, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v4, v6, v10, s[4:5]
+; SDAG-NEXT:    v_ffbh_u32_e32 v10, v8
+; SDAG-NEXT:    v_ffbh_u32_e32 v11, v9
+; SDAG-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; SDAG-NEXT:    v_sub_i32_e32 v19, vcc, 0, v12
+; SDAG-NEXT:    v_or_b32_e32 v6, v8, v4
+; SDAG-NEXT:    v_ffbh_u32_e32 v20, v4
+; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], 32, v10
+; SDAG-NEXT:    v_subb_u32_e32 v21, vcc, 0, v13, vcc
+; SDAG-NEXT:    v_or_b32_e32 v7, v9, v5
+; SDAG-NEXT:    v_add_i32_e64 v20, s[4:5], 32, v20
+; SDAG-NEXT:    v_ffbh_u32_e32 v22, v5
+; SDAG-NEXT:    v_min_u32_e32 v10, v10, v11
+; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, 0, v14, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[14:15]
+; SDAG-NEXT:    v_cndmask_b32_e64 v36, v13, v21, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v37, v12, v19, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], 0, v[6:7]
+; SDAG-NEXT:    v_min_u32_e32 v7, v20, v22
+; SDAG-NEXT:    v_add_i32_e64 v10, s[8:9], 64, v10
+; SDAG-NEXT:    v_addc_u32_e64 v12, s[8:9], 0, 0, s[8:9]
+; SDAG-NEXT:    v_subb_u32_e32 v13, vcc, 0, v15, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v6, v14, v11, s[4:5]
 ; SDAG-NEXT:    v_ffbh_u32_e32 v11, v37
-; SDAG-NEXT:    v_min_u32_e32 v12, v12, v19
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v19, v20, 0, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v10, vcc, v14, v16, vcc
-; SDAG-NEXT:    v_add_i32_e64 v13, s[8:9], 32, v11
 ; SDAG-NEXT:    v_ffbh_u32_e32 v14, v36
-; SDAG-NEXT:    v_cndmask_b32_e64 v18, v18, v12, s[6:7]
-; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, v15, v16, vcc
-; SDAG-NEXT:    v_or_b32_e32 v12, v37, v10
-; SDAG-NEXT:    v_ffbh_u32_e32 v15, v10
-; SDAG-NEXT:    v_min_u32_e32 v14, v13, v14
-; SDAG-NEXT:    v_or_b32_e32 v13, v36, v11
-; SDAG-NEXT:    v_add_i32_e32 v15, vcc, 32, v15
-; SDAG-NEXT:    v_ffbh_u32_e32 v16, v11
-; SDAG-NEXT:    v_add_i32_e32 v14, vcc, 64, v14
-; SDAG-NEXT:    v_addc_u32_e64 v20, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, v12, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v19, v10, v7, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v7, v15, v13, s[4:5]
+; SDAG-NEXT:    v_or_b32_e32 v10, v37, v6
+; SDAG-NEXT:    v_ffbh_u32_e32 v13, v6
+; SDAG-NEXT:    v_add_i32_e32 v15, vcc, 32, v11
+; SDAG-NEXT:    v_or_b32_e32 v11, v36, v7
+; SDAG-NEXT:    v_add_i32_e32 v13, vcc, 32, v13
+; SDAG-NEXT:    v_ffbh_u32_e32 v20, v7
+; SDAG-NEXT:    v_min_u32_e32 v14, v15, v14
+; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; SDAG-NEXT:    v_min_u32_e32 v10, v13, v20
+; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], 64, v14
+; SDAG-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; SDAG-NEXT:    v_cndmask_b32_e64 v13, v13, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc
+; SDAG-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; SDAG-NEXT:    v_subb_u32_e32 v11, vcc, v13, v12, vcc
+; SDAG-NEXT:    v_xor_b32_e32 v14, 0x7f, v10
+; SDAG-NEXT:    v_subbrev_u32_e32 v12, vcc, 0, v18, vcc
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[10:11]
+; SDAG-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[4:5]
+; SDAG-NEXT:    v_subbrev_u32_e32 v13, vcc, 0, v18, vcc
+; SDAG-NEXT:    v_or_b32_e32 v14, v14, v12
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; SDAG-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; SDAG-NEXT:    v_or_b32_e32 v15, v11, v13
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT:    v_min_u32_e32 v12, v15, v16
-; SDAG-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[10:11]
-; SDAG-NEXT:    v_cndmask_b32_e64 v13, v20, 0, s[6:7]
-; SDAG-NEXT:    s_or_b64 s[8:9], vcc, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v12, v14, v12, s[6:7]
-; SDAG-NEXT:    v_sub_i32_e32 v12, vcc, v12, v18
-; SDAG-NEXT:    v_subb_u32_e32 v13, vcc, v13, v19, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v16, 0x7f, v12
-; SDAG-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v17, vcc
-; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[10:11], v[12:13]
-; SDAG-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
-; SDAG-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v17, vcc
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v14
+; SDAG-NEXT:    v_cndmask_b32_e32 v18, v18, v19, vcc
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v13, v15
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[16:17]
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v18
-; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
-; SDAG-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; SDAG-NEXT:    v_and_b32_e32 v14, 1, v18
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v14
+; SDAG-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v19, v5, 0, s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
 ; SDAG-NEXT:    v_cndmask_b32_e64 v18, v4, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v7, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v16, v6, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v15, v9, 0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v14, v8, 0, s[4:5]
 ; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_12
 ; SDAG-NEXT:  ; %bb.7: ; %udiv-bb1
-; SDAG-NEXT:    v_add_i32_e32 v38, vcc, 1, v12
-; SDAG-NEXT:    v_sub_i32_e64 v18, s[4:5], 63, v12
-; SDAG-NEXT:    v_mov_b32_e32 v16, 0
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
-; SDAG-NEXT:    v_addc_u32_e32 v39, vcc, 0, v13, vcc
-; SDAG-NEXT:    v_lshl_b64 v[18:19], v[6:7], v18
-; SDAG-NEXT:    v_addc_u32_e32 v48, vcc, 0, v14, vcc
-; SDAG-NEXT:    v_addc_u32_e32 v49, vcc, 0, v15, vcc
-; SDAG-NEXT:    v_or_b32_e32 v13, v38, v48
-; SDAG-NEXT:    v_sub_i32_e32 v15, vcc, 0x7f, v12
-; SDAG-NEXT:    v_or_b32_e32 v14, v39, v49
-; SDAG-NEXT:    v_lshl_b64 v[20:21], v[4:5], v15
-; SDAG-NEXT:    v_sub_i32_e32 v12, vcc, 64, v15
-; SDAG-NEXT:    v_lshl_b64 v[22:23], v[6:7], v15
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[13:14]
-; SDAG-NEXT:    v_lshr_b64 v[12:13], v[6:7], v12
-; SDAG-NEXT:    v_or_b32_e32 v13, v21, v13
-; SDAG-NEXT:    v_or_b32_e32 v12, v20, v12
-; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v15
-; SDAG-NEXT:    v_cndmask_b32_e64 v14, v19, v13, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v18, v18, v12, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v23, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v22, s[4:5]
-; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v15
-; SDAG-NEXT:    v_cndmask_b32_e64 v15, v14, v5, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v14, v18, v4, s[4:5]
+; SDAG-NEXT:    v_add_i32_e32 v38, vcc, 1, v10
+; SDAG-NEXT:    v_sub_i32_e64 v18, s[4:5], 63, v10
+; SDAG-NEXT:    v_mov_b32_e32 v14, 0
+; SDAG-NEXT:    v_mov_b32_e32 v15, 0
+; SDAG-NEXT:    v_addc_u32_e32 v39, vcc, 0, v11, vcc
+; SDAG-NEXT:    v_lshl_b64 v[18:19], v[8:9], v18
+; SDAG-NEXT:    v_addc_u32_e32 v48, vcc, 0, v12, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v49, vcc, 0, v13, vcc
+; SDAG-NEXT:    v_or_b32_e32 v11, v38, v48
+; SDAG-NEXT:    v_sub_i32_e32 v13, vcc, 0x7f, v10
+; SDAG-NEXT:    v_or_b32_e32 v12, v39, v49
+; SDAG-NEXT:    v_lshl_b64 v[20:21], v[4:5], v13
+; SDAG-NEXT:    v_sub_i32_e32 v10, vcc, 64, v13
+; SDAG-NEXT:    v_lshl_b64 v[22:23], v[8:9], v13
+; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[11:12]
+; SDAG-NEXT:    v_lshr_b64 v[10:11], v[8:9], v10
+; SDAG-NEXT:    v_or_b32_e32 v11, v21, v11
+; SDAG-NEXT:    v_or_b32_e32 v10, v20, v10
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v13
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, v19, v11, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v18, v18, v10, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, 0, v23, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, v22, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v13
+; SDAG-NEXT:    v_cndmask_b32_e64 v13, v12, v5, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, v18, v4, s[4:5]
 ; SDAG-NEXT:    v_mov_b32_e32 v18, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v19, 0
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_11
 ; SDAG-NEXT:  ; %bb.8: ; %udiv-preheader
-; SDAG-NEXT:    v_lshr_b64 v[16:17], v[6:7], v38
+; SDAG-NEXT:    v_lshr_b64 v[14:15], v[8:9], v38
 ; SDAG-NEXT:    v_sub_i32_e32 v24, vcc, 64, v38
 ; SDAG-NEXT:    v_subrev_i32_e32 v51, vcc, 64, v38
 ; SDAG-NEXT:    v_lshr_b64 v[22:23], v[4:5], v38
@@ -1871,42 +1877,42 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_lshl_b64 v[24:25], v[4:5], v24
 ; SDAG-NEXT:    v_lshr_b64 v[53:54], v[4:5], v51
 ; SDAG-NEXT:    v_addc_u32_e32 v51, vcc, -1, v36, vcc
-; SDAG-NEXT:    v_or_b32_e32 v17, v17, v25
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v24
-; SDAG-NEXT:    v_addc_u32_e32 v52, vcc, -1, v10, vcc
+; SDAG-NEXT:    v_or_b32_e32 v15, v15, v25
+; SDAG-NEXT:    v_or_b32_e32 v14, v14, v24
+; SDAG-NEXT:    v_addc_u32_e32 v52, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v38
-; SDAG-NEXT:    v_cndmask_b32_e64 v17, v54, v17, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v16, v53, v16, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v15, v54, v15, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v14, v53, v14, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v25, 0, v23, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v24, 0, v22, s[4:5]
-; SDAG-NEXT:    v_addc_u32_e32 v53, vcc, -1, v11, vcc
+; SDAG-NEXT:    v_addc_u32_e32 v53, vcc, -1, v7, vcc
 ; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v38
-; SDAG-NEXT:    v_cndmask_b32_e32 v23, v17, v7, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v22, v16, v6, vcc
-; SDAG-NEXT:    v_mov_b32_e32 v17, 0
+; SDAG-NEXT:    v_cndmask_b32_e32 v23, v15, v9, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v22, v14, v8, vcc
+; SDAG-NEXT:    v_mov_b32_e32 v15, 0
 ; SDAG-NEXT:  .LBB2_9: ; %udiv-do-while
 ; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SDAG-NEXT:    v_lshl_b64 v[24:25], v[24:25], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v16, 31, v23
+; SDAG-NEXT:    v_lshrrev_b32_e32 v14, 31, v23
 ; SDAG-NEXT:    v_lshl_b64 v[22:23], v[22:23], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v54, 31, v15
-; SDAG-NEXT:    v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v55, 31, v13
+; SDAG-NEXT:    v_lshrrev_b32_e32 v54, 31, v13
 ; SDAG-NEXT:    v_lshl_b64 v[12:13], v[12:13], 1
-; SDAG-NEXT:    v_or_b32_e32 v24, v24, v16
+; SDAG-NEXT:    v_lshrrev_b32_e32 v55, 31, v11
+; SDAG-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; SDAG-NEXT:    v_or_b32_e32 v24, v24, v14
 ; SDAG-NEXT:    v_or_b32_e32 v22, v22, v54
-; SDAG-NEXT:    v_or_b32_e32 v14, v14, v55
-; SDAG-NEXT:    v_or_b32_e32 v15, v19, v15
-; SDAG-NEXT:    v_or_b32_e32 v13, v21, v13
-; SDAG-NEXT:    v_or_b32_e32 v14, v18, v14
-; SDAG-NEXT:    v_sub_i32_e32 v16, vcc, v50, v22
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v51, v23, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v52, v24, vcc
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v53, v25, vcc
-; SDAG-NEXT:    v_ashrrev_i32_e32 v21, 31, v16
-; SDAG-NEXT:    v_and_b32_e32 v16, 1, v21
-; SDAG-NEXT:    v_and_b32_e32 v54, v21, v11
-; SDAG-NEXT:    v_and_b32_e32 v55, v21, v10
+; SDAG-NEXT:    v_or_b32_e32 v12, v12, v55
+; SDAG-NEXT:    v_or_b32_e32 v13, v19, v13
+; SDAG-NEXT:    v_or_b32_e32 v11, v21, v11
+; SDAG-NEXT:    v_or_b32_e32 v12, v18, v12
+; SDAG-NEXT:    v_sub_i32_e32 v14, vcc, v50, v22
+; SDAG-NEXT:    v_subb_u32_e32 v14, vcc, v51, v23, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v14, vcc, v52, v24, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v14, vcc, v53, v25, vcc
+; SDAG-NEXT:    v_ashrrev_i32_e32 v21, 31, v14
+; SDAG-NEXT:    v_and_b32_e32 v14, 1, v21
+; SDAG-NEXT:    v_and_b32_e32 v54, v21, v7
+; SDAG-NEXT:    v_and_b32_e32 v55, v21, v6
 ; SDAG-NEXT:    v_and_b32_e32 v40, v21, v36
 ; SDAG-NEXT:    v_and_b32_e32 v21, v21, v37
 ; SDAG-NEXT:    v_sub_i32_e32 v22, vcc, v22, v21
@@ -1921,89 +1927,87 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; SDAG-NEXT:    v_or_b32_e32 v54, v38, v48
 ; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[54:55]
 ; SDAG-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; SDAG-NEXT:    v_or_b32_e32 v12, v20, v12
-; SDAG-NEXT:    v_mov_b32_e32 v21, v17
-; SDAG-NEXT:    v_mov_b32_e32 v20, v16
+; SDAG-NEXT:    v_or_b32_e32 v10, v20, v10
+; SDAG-NEXT:    v_mov_b32_e32 v21, v15
+; SDAG-NEXT:    v_mov_b32_e32 v20, v14
 ; SDAG-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; SDAG-NEXT:    s_cbranch_execnz .LBB2_9
 ; SDAG-NEXT:  ; %bb.10: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; SDAG-NEXT:  .LBB2_11: ; %Flow11
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT:    v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT:    v_lshrrev_b32_e32 v20, 31, v13
 ; SDAG-NEXT:    v_lshl_b64 v[12:13], v[12:13], 1
-; SDAG-NEXT:    v_or_b32_e32 v14, v14, v20
-; SDAG-NEXT:    v_or_b32_e32 v19, v19, v15
-; SDAG-NEXT:    v_or_b32_e32 v17, v17, v13
-; SDAG-NEXT:    v_or_b32_e32 v18, v18, v14
-; SDAG-NEXT:    v_or_b32_e32 v16, v16, v12
+; SDAG-NEXT:    v_lshrrev_b32_e32 v20, 31, v11
+; SDAG-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; SDAG-NEXT:    v_or_b32_e32 v12, v12, v20
+; SDAG-NEXT:    v_or_b32_e32 v19, v19, v13
+; SDAG-NEXT:    v_or_b32_e32 v15, v15, v11
+; SDAG-NEXT:    v_or_b32_e32 v18, v18, v12
+; SDAG-NEXT:    v_or_b32_e32 v14, v14, v10
 ; SDAG-NEXT:  .LBB2_12: ; %Flow12
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT:    v_mul_lo_u32 v14, v33, v9
-; SDAG-NEXT:    v_mad_u64_u32 v[12:13], s[4:5], v33, v8, 0
-; SDAG-NEXT:    v_mul_lo_u32 v24, v27, v8
-; SDAG-NEXT:    v_mul_lo_u32 v25, v35, v31
-; SDAG-NEXT:    v_mul_lo_u32 v35, v32, v30
-; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v31, v33, 0
-; SDAG-NEXT:    v_mov_b32_e32 v15, 0
-; SDAG-NEXT:    v_mul_lo_u32 v38, v16, v11
-; SDAG-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v16, v10, 0
-; SDAG-NEXT:    v_mul_lo_u32 v39, v17, v10
+; SDAG-NEXT:    v_mul_lo_u32 v12, v33, v3
+; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v33, v2, 0
+; SDAG-NEXT:    v_mul_lo_u32 v24, v27, v2
+; SDAG-NEXT:    v_mul_lo_u32 v35, v35, v31
+; SDAG-NEXT:    v_mul_lo_u32 v38, v32, v30
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v31, v33, 0
+; SDAG-NEXT:    v_mov_b32_e32 v13, 0
+; SDAG-NEXT:    v_mul_lo_u32 v25, v14, v7
+; SDAG-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v14, v6, 0
+; SDAG-NEXT:    v_mul_lo_u32 v39, v15, v6
 ; SDAG-NEXT:    v_mul_lo_u32 v19, v19, v37
 ; SDAG-NEXT:    v_mul_lo_u32 v48, v18, v36
-; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v37, v16, 0
-; SDAG-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
-; SDAG-NEXT:    v_mov_b32_e32 v14, v9
-; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v30, v33, v[14:15]
-; SDAG-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
-; SDAG-NEXT:    v_add_i32_e64 v14, s[4:5], v21, v38
-; SDAG-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v24
+; SDAG-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v37, v14, 0
+; SDAG-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; SDAG-NEXT:    v_mov_b32_e32 v12, v3
+; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v30, v33, v[12:13]
+; SDAG-NEXT:    v_sub_i32_e32 v12, vcc, v16, v2
+; SDAG-NEXT:    v_add_i32_e64 v16, s[4:5], v21, v25
+; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v24
 ; SDAG-NEXT:    v_mov_b32_e32 v24, v23
-; SDAG-NEXT:    v_mov_b32_e32 v23, v15
-; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v31, v27, v[22:23]
-; SDAG-NEXT:    v_xor_b32_e32 v33, v2, v28
-; SDAG-NEXT:    v_add_i32_e64 v21, s[4:5], v14, v39
-; SDAG-NEXT:    v_mov_b32_e32 v14, v11
-; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v36, v16, v[14:15]
-; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v32, v31, v[12:13]
-; SDAG-NEXT:    v_mov_b32_e32 v2, v9
-; SDAG-NEXT:    v_add_i32_e64 v13, s[4:5], v24, v2
-; SDAG-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v2, v8
-; SDAG-NEXT:    v_subb_u32_e32 v16, vcc, v3, v2, vcc
+; SDAG-NEXT:    v_mov_b32_e32 v23, v13
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v31, v27, v[22:23]
+; SDAG-NEXT:    v_xor_b32_e32 v33, v12, v28
+; SDAG-NEXT:    v_add_i32_e64 v21, s[4:5], v16, v39
+; SDAG-NEXT:    v_mov_b32_e32 v12, v7
+; SDAG-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v36, v14, v[12:13]
+; SDAG-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v32, v31, v[10:11]
+; SDAG-NEXT:    v_add_i32_e64 v24, s[4:5], v24, v3
+; SDAG-NEXT:    v_addc_u32_e64 v25, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v17, v2, vcc
 ; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v18, v37, v[20:21]
-; SDAG-NEXT:    v_mov_b32_e32 v18, v23
-; SDAG-NEXT:    v_mov_b32_e32 v23, v15
-; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v37, v17, v[22:23]
-; SDAG-NEXT:    v_add_i32_e64 v20, s[4:5], v25, v12
-; SDAG-NEXT:    v_mad_u64_u32 v[12:13], s[4:5], v30, v27, v[13:14]
-; SDAG-NEXT:    v_xor_b32_e32 v16, v16, v29
+; SDAG-NEXT:    v_mov_b32_e32 v14, v23
+; SDAG-NEXT:    v_mov_b32_e32 v23, v13
+; SDAG-NEXT:    v_mad_u64_u32 v[12:13], s[4:5], v37, v15, v[22:23]
+; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v35, v11
+; SDAG-NEXT:    v_mad_u64_u32 v[16:17], s[4:5], v30, v27, v[24:25]
+; SDAG-NEXT:    v_xor_b32_e32 v7, v7, v29
 ; SDAG-NEXT:    v_add_i32_e64 v3, s[4:5], v19, v3
-; SDAG-NEXT:    v_add_i32_e64 v14, s[4:5], v18, v9
-; SDAG-NEXT:    v_addc_u32_e64 v15, s[4:5], 0, 0, s[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v18, v8
-; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], v35, v20
+; SDAG-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; SDAG-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mov_b32_e32 v18, v12
+; SDAG-NEXT:    v_add_i32_e64 v19, s[4:5], v38, v11
 ; SDAG-NEXT:    v_add_i32_e64 v3, s[4:5], v48, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v36, v17, v[14:15]
-; SDAG-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
-; SDAG-NEXT:    v_addc_u32_e64 v12, s[4:5], v13, v19, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v0, v11, vcc
-; SDAG-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v2
-; SDAG-NEXT:    v_addc_u32_e64 v9, s[4:5], v9, v3, s[4:5]
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v36, v15, v[13:14]
+; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], v16, v10
+; SDAG-NEXT:    v_addc_u32_e64 v13, s[4:5], v17, v19, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v0, vcc, v0, v10, vcc
+; SDAG-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v2
+; SDAG-NEXT:    v_addc_u32_e64 v11, s[4:5], v12, v3, s[4:5]
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v2, v0, v28
 ; SDAG-NEXT:    v_xor_b32_e32 v3, v1, v29
 ; SDAG-NEXT:    v_sub_i32_e32 v0, vcc, v33, v28
-; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v16, v29, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v1, vcc, v7, v29, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v2, vcc, v2, v28, vcc
 ; SDAG-NEXT:    v_subb_u32_e32 v3, vcc, v3, v29, vcc
-; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v6, v10
-; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v7, v18, vcc
+; SDAG-NEXT:    v_sub_i32_e32 v6, vcc, v8, v6
+; SDAG-NEXT:    v_subb_u32_e32 v7, vcc, v9, v18, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v6, v6, v26
-; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v4, v8, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v4, vcc, v4, v10, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v7, v7, v34
-; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v9, vcc
+; SDAG-NEXT:    v_subb_u32_e32 v5, vcc, v5, v11, vcc
 ; SDAG-NEXT:    v_xor_b32_e32 v8, v4, v26
 ; SDAG-NEXT:    v_xor_b32_e32 v9, v5, v34
 ; SDAG-NEXT:    v_sub_i32_e32 v4, vcc, v6, v26
diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
index edc20153ffd770d11f022c58a3885f3b5cd4035b..560a05abd5e713315c8188ce663e65d7c1011a4b 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
@@ -74,6 +74,7 @@
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1103 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1103 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1150 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1150 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1151 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1151 %s
+; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1152 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1152 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1200 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1200 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1201 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1201 %s
 
@@ -153,6 +154,7 @@
 ; GFX1103:       EF_AMDGPU_MACH_AMDGCN_GFX1103 (0x44)
 ; GFX1150:       EF_AMDGPU_MACH_AMDGCN_GFX1150 (0x43)
 ; GFX1151:       EF_AMDGPU_MACH_AMDGCN_GFX1151 (0x4A)
+; GFX1152:       EF_AMDGPU_MACH_AMDGCN_GFX1152 (0x55)
 ; GFX1200:       EF_AMDGPU_MACH_AMDGCN_GFX1200 (0x48)
 ; GFX1201:       EF_AMDGPU_MACH_AMDGCN_GFX1201 (0x4E)
 
diff --git a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ce55558dabaf13b47e30fd76a5184ca053ae2de3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
@@ -0,0 +1,545 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
+; RUN: opt -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s
+; REQUIRES: amdgpu-registered-target
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; Check the variables are lowered to the locations this target expects
+
+; The types show the call frames
+; CHECK: %single_i32.vararg = type <{ i32 }>
+; CHECK: %single_double.vararg = type <{ double }>
+; CHECK: %single_v4f32.vararg = type <{ <4 x float> }>
+; CHECK: %single_v8f32.vararg = type <{ <8 x float> }>
+; CHECK: %single_v16f32.vararg = type <{ <16 x float> }>
+; CHECK: %single_v32f32.vararg = type <{ <32 x float> }>
+; CHECK: %i32_double.vararg = type <{ i32, double }>
+; CHECK: %double_i32.vararg = type <{ double, i32 }>
+; CHECK: %i32_libcS.vararg = type <{ i32, %struct.libcS }>
+; CHECK: %libcS_i32.vararg = type <{ %struct.libcS, i32 }>
+; CHECK: %i32_v4f32.vararg = type <{ i32, <4 x float> }>
+; CHECK: %v4f32_i32.vararg = type <{ <4 x float>, i32 }>
+; CHECK: %i32_v8f32.vararg = type <{ i32, <8 x float> }>
+; CHECK: %v8f32_i32.vararg = type <{ <8 x float>, i32 }>
+; CHECK: %i32_v16f32.vararg = type <{ i32, <16 x float> }>
+; CHECK: %v16f32_i32.vararg = type <{ <16 x float>, i32 }>
+; CHECK: %i32_v32f32.vararg = type <{ i32, <32 x float> }>
+; CHECK: %v32f32_i32.vararg = type <{ <32 x float>, i32 }>
+; CHECK: %fptr_single_i32.vararg = type <{ i32 }>
+; CHECK: %fptr_libcS.vararg = type <{ %struct.libcS }>
+
+%struct.libcS = type { i8, i16, i32, i64, float, double }
+
+@vararg_ptr = hidden addrspace(1) global ptr @vararg, align 8
+
+define hidden void @copy(ptr noundef %va) {
+; CHECK-LABEL: define {{[^@]+}}@copy(ptr noundef %va) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %va.addr = alloca ptr, align 8, addrspace(5)
+; CHECK-NEXT:    %cp = alloca ptr, align 8, addrspace(5)
+; CHECK-NEXT:    %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr
+; CHECK-NEXT:    %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr
+; CHECK-NEXT:    store ptr %va, ptr addrspace(5) %va.addr, align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr %cp.ascast, ptr %va.addr.ascast, i32 8, i1 false)
+; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %cp, align 8
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %va.addr = alloca ptr, align 8, addrspace(5)
+  %cp = alloca ptr, align 8, addrspace(5)
+  %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr
+  %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr
+  store ptr %va, ptr addrspace(5) %va.addr, align 8
+  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp)
+  call void @llvm.va_copy.p0(ptr %cp.ascast, ptr nonnull %va.addr.ascast)
+  %0 = load ptr, ptr addrspace(5) %cp, align 8
+  call void @valist(ptr noundef %0)
+  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture)
+
+declare void @llvm.va_copy.p0(ptr, ptr)
+
+declare hidden void @valist(ptr noundef)
+
+declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture)
+
+define hidden void @start_once(...) {
+; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %s = alloca ptr, align 8, addrspace(5)
+; CHECK-NEXT:    %s.ascast = addrspacecast ptr addrspace(5) %s to ptr
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s)
+; CHECK-NEXT:    store ptr %varargs, ptr %s.ascast, align 8
+; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %s, align 8
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %s = alloca ptr, align 8, addrspace(5)
+  %s.ascast = addrspacecast ptr addrspace(5) %s to ptr
+  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s)
+  call void @llvm.va_start.p0(ptr %s.ascast)
+  %0 = load ptr, ptr addrspace(5) %s, align 8
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s.ascast)
+  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s)
+  ret void
+}
+
+declare void @llvm.va_start.p0(ptr)
+
+declare void @llvm.va_end.p0(ptr)
+
+define hidden void @start_twice(...) {
+; CHECK-LABEL: define {{[^@]+}}@start_twice(ptr %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %s0 = alloca ptr, align 8, addrspace(5)
+; CHECK-NEXT:    %s1 = alloca ptr, align 8, addrspace(5)
+; CHECK-NEXT:    %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr
+; CHECK-NEXT:    %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1)
+; CHECK-NEXT:    store ptr %varargs, ptr %s0.ascast, align 8
+; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %s0, align 8
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    store ptr %varargs, ptr %s1.ascast, align 8
+; CHECK-NEXT:    %1 = load ptr, ptr addrspace(5) %s1, align 8
+; CHECK-NEXT:    call void @valist(ptr noundef %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %s0 = alloca ptr, align 8, addrspace(5)
+  %s1 = alloca ptr, align 8, addrspace(5)
+  %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr
+  %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr
+  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0)
+  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1)
+  call void @llvm.va_start.p0(ptr %s0.ascast)
+  %0 = load ptr, ptr addrspace(5) %s0, align 8
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s0.ascast)
+  call void @llvm.va_start.p0(ptr %s1.ascast)
+  %1 = load ptr, ptr addrspace(5) %s1, align 8
+  call void @valist(ptr noundef %1)
+  call void @llvm.va_end.p0(ptr %s1.ascast)
+  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1)
+  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0)
+  ret void
+}
+
+define hidden void @single_i32(i32 noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x)
+  ret void
+}
+
+declare hidden void @vararg(...)
+
+define hidden void @single_double(double noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_double.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store double %x, ptr addrspace(5) %0, align 8
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(double noundef %x)
+  ret void
+}
+
+define hidden void @single_v4f32(<4 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v4f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 16, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <4 x float> %x, ptr addrspace(5) %0, align 16
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 16, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<4 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v8f32(<8 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v8f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <8 x float> %x, ptr addrspace(5) %0, align 32
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<8 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v16f32(<16 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v16f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 64, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <16 x float> %x, ptr addrspace(5) %0, align 64
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 64, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<16 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v32f32(<32 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v32f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 128, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <32 x float> %x, ptr addrspace(5) %0, align 128
+; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 128, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<32 x float> noundef %x)
+  ret void
+}
+
+define hidden void @i32_double(i32 noundef %x, double noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_double.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store double %y, ptr addrspace(5) %1, align 8
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, double noundef %y)
+  ret void
+}
+
+define hidden void @double_i32(double noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %double_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store double %x, ptr addrspace(5) %0, align 8
+; CHECK-NEXT:    %1 = getelementptr inbounds %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(double noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_libcS(i32 noundef %x, i8 %y.coerce0, i16 %y.coerce1, i32 %y.coerce2, i64 %y.coerce3, float %y.coerce4, double %y.coerce5) {
+; CHECK-LABEL: define {{[^@]+}}@i32_libcS(i32 noundef %x, i8 %y.coerce0, i16 %y.coerce1, i32 %y.coerce2, i64 %y.coerce3, float %y.coerce4, double %y.coerce5) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_libcS.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    %.fca.0.insert = insertvalue %struct.libcS poison, i8 %y.coerce0, 0
+; CHECK-NEXT:    %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %y.coerce1, 1
+; CHECK-NEXT:    %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %y.coerce2, 2
+; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %y.coerce3, 3
+; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %y.coerce4, 4
+; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %y.coerce5, 5
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %.fca.0.insert = insertvalue %struct.libcS poison, i8 %y.coerce0, 0
+  %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %y.coerce1, 1
+  %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %y.coerce2, 2
+  %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %y.coerce3, 3
+  %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %y.coerce4, 4
+  %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %y.coerce5, 5
+  tail call void (...) @vararg(i32 noundef %x, %struct.libcS %.fca.5.insert)
+  ret void
+}
+
+define hidden void @libcS_i32(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@libcS_i32(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %libcS_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0
+; CHECK-NEXT:    %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1
+; CHECK-NEXT:    %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2
+; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
+; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
+; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %0, align 8
+; CHECK-NEXT:    %1 = getelementptr inbounds %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0
+  %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1
+  %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2
+  %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
+  %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
+  %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
+  tail call void (...) @vararg(%struct.libcS %.fca.5.insert, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v4f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store <4 x float> %y, ptr addrspace(5) %1, align 16
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <4 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v4f32_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <4 x float> %x, ptr addrspace(5) %0, align 16
+; CHECK-NEXT:    %1 = getelementptr inbounds %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<4 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v8f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store <8 x float> %y, ptr addrspace(5) %1, align 32
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <8 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v8f32_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <8 x float> %x, ptr addrspace(5) %0, align 32
+; CHECK-NEXT:    %1 = getelementptr inbounds %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<8 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v16f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store <16 x float> %y, ptr addrspace(5) %1, align 64
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <16 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v16f32_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <16 x float> %x, ptr addrspace(5) %0, align 64
+; CHECK-NEXT:    %1 = getelementptr inbounds %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<16 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v32f32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store <32 x float> %y, ptr addrspace(5) %1, align 128
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <32 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v32f32_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <32 x float> %x, ptr addrspace(5) %0, align 128
+; CHECK-NEXT:    %1 = getelementptr inbounds %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void @vararg(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<32 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @fptr_single_i32(i32 noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@fptr_single_i32(i32 noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %fptr_single_i32.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %1 = getelementptr inbounds %fptr_single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %1, align 4
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void %0(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8
+  tail call void (...) %0(i32 noundef %x)
+  ret void
+}
+
+define hidden void @fptr_libcS(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5) {
+; CHECK-LABEL: define {{[^@]+}}@fptr_libcS(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %fptr_libcS.vararg, align 4, addrspace(5)
+; CHECK-NEXT:    %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8
+; CHECK-NEXT:    %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0
+; CHECK-NEXT:    %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1
+; CHECK-NEXT:    %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2
+; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
+; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
+; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    %1 = getelementptr inbounds %fptr_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8
+; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
+; CHECK-NEXT:    call void %0(ptr %2)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8
+  %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0
+  %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1
+  %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2
+  %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
+  %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
+  %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
+  tail call void (...) %0(%struct.libcS %.fca.5.insert)
+  ret void
+}
+
+
diff --git a/llvm/test/CodeGen/AMDGPU/fabs-r600.ll b/llvm/test/CodeGen/AMDGPU/fabs-r600.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7e1aa99c3ec40f97ceacdb6db45ff27844087a57
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fabs-r600.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 %s
+
+
+; DAGCombiner will transform:
+; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF))
+; unless isFabsFree returns true
+define amdgpu_kernel void @s_fabsf_fn_free(ptr addrspace(1) %out, i32 %in) {
+; R600-LABEL: s_fabsf_fn_free:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV T0.X, |PV.W|,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %bc= bitcast i32 %in to float
+  %fabs = call float @fabsf(float %bc)
+  store float %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) {
+; R600-LABEL: s_fabsf_free:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV T0.X, |PV.W|,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %bc= bitcast i32 %in to float
+  %fabs = call float @llvm.fabs.f32(float %bc)
+  store float %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; R600-LABEL: s_fabsf_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV T0.X, |PV.W|,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @llvm.fabs.f32(float %in)
+  store float %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
+; R600-LABEL: fabs_v2f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 5, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[3].X,
+; R600-NEXT:     MOV T0.Y, |PV.W|,
+; R600-NEXT:     MOV * T0.W, KC0[2].W,
+; R600-NEXT:     MOV T0.X, |PV.W|,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
+  store <2 x float> %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
+; R600-LABEL: fabsf_v4f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 9, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV T0.W, KC0[4].X,
+; R600-NEXT:     MOV * T1.W, KC0[3].W,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T0.Z, |T1.W|,
+; R600-NEXT:     MOV * T1.W, KC0[3].Z,
+; R600-NEXT:     MOV T0.Y, |PV.W|,
+; R600-NEXT:     MOV * T1.W, KC0[3].Y,
+; R600-NEXT:     MOV T0.X, |PV.W|,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
+  store <4 x float> %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fabsf_fn_fold(ptr addrspace(1) %out, float %in0, float %in1) {
+; R600-LABEL: fabsf_fn_fold:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MUL_IEEE * T1.X, |KC0[2].Z|, KC0[2].W,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @fabsf(float %in0)
+  %fmul = fmul float %fabs, %in1
+  store float %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %in1) {
+; R600-LABEL: fabs_fold:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MUL_IEEE * T1.X, |KC0[2].Z|, KC0[2].W,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @llvm.fabs.f32(float %in0)
+  %fmul = fmul float %fabs, %in1
+  store float %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @bitpreserve_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; R600-LABEL: bitpreserve_fabsf_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     ADD * T1.X, |KC0[2].Z|, 1.0,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %in.bc = bitcast float %in to i32
+  %int.abs = and i32 %in.bc, 2147483647
+  %bc = bitcast i32 %int.abs to float
+  %fadd = fadd float %bc, 1.0
+  store float %fadd, ptr addrspace(1) %out
+  ret void
+}
+
+declare float @fabsf(float) readnone
+declare float @llvm.fabs.f32(float) readnone
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll
index e18c76f89b6c7fe8d6e80395e15b08d9aaa299fc..07581ade57ccd595d646c47abf140729056d14bb 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.ll
@@ -1,104 +1,256 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs -enable-misched=0 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global,-xnack -enable-misched=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=amdgcn -enable-misched=0 < %s | FileCheck -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -enable-misched=0 < %s | FileCheck -check-prefixes=GCN,VI %s
 
 
 ; DAGCombiner will transform:
 ; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF))
 ; unless isFabsFree returns true
-
-; FUNC-LABEL: {{^}}s_fabsf_fn_free:
-; R600-NOT: AND
-; R600: |PV.{{[XYZW]}}|
-
-; GCN: s_bitset0_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @s_fabsf_fn_free(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: s_fabsf_fn_free:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x9
+; SI-NEXT:    s_load_dword s4, s[2:3], 0xb
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset0_b32 s4, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabsf_fn_free:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x24
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_bitset0_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %bc= bitcast i32 %in to float
   %fabs = call float @fabsf(float %bc)
   store float %fabs, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_fabsf_free:
-; R600-NOT: AND
-; R600: |PV.{{[XYZW]}}|
-
-; GCN: s_bitset0_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: s_fabsf_free:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset0_b32 s0, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabsf_free:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    s_bitset0_b32 s0, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %bc= bitcast i32 %in to float
   %fabs = call float @llvm.fabs.f32(float %bc)
   store float %fabs, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_fabsf_f32:
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-
-; GCN: s_bitset0_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @s_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: s_fabsf_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset0_b32 s0, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabsf_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    s_bitset0_b32 s0, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @llvm.fabs.f32(float %in)
   store float %fabs, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}fabs_v2f32:
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-
-; GCN: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x7fffffff
-; GCN: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x7fffffff
 define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
+; SI-LABEL: fabs_v2f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_and_b32 s0, s3, 0x7fffffff
+; SI-NEXT:    s_and_b32 s1, s2, 0x7fffffff
+; SI-NEXT:    v_mov_b32_e32 v0, s1
+; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabs_v2f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset0_b32 s3, 31
+; VI-NEXT:    s_bitset0_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   store <2 x float> %fabs, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}fabsf_v4f32:
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-
-; GCN: s_bitset0_b32
-; GCN: s_bitset0_b32
-; GCN: s_bitset0_b32
-; GCN: s_bitset0_b32
 define amdgpu_kernel void @fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
+; SI-LABEL: fabsf_v4f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset0_b32 s3, 31
+; SI-NEXT:    s_bitset0_b32 s2, 31
+; SI-NEXT:    s_bitset0_b32 s1, 31
+; SI-NEXT:    s_bitset0_b32 s0, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    v_mov_b32_e32 v2, s2
+; SI-NEXT:    v_mov_b32_e32 v3, s3
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabsf_v4f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_bitset0_b32 s3, 31
+; VI-NEXT:    s_bitset0_b32 s2, 31
+; VI-NEXT:    s_bitset0_b32 s1, 31
+; VI-NEXT:    s_bitset0_b32 s0, 31
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
   %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
   store <4 x float> %fabs, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}fabsf_fn_fold:
-; SI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x9
-; VI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x24
-; GCN-NOT: and
-; GCN: v_mov_b32_e32 [[V_MUL_VI:v[0-9]+]], s[[#LOAD + 3]]
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, |s[[#LOAD + 2]]|, [[V_MUL_VI]]
 define amdgpu_kernel void @fabsf_fn_fold(ptr addrspace(1) %out, float %in0, float %in1) {
+; SI-LABEL: fabsf_fn_fold:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s3
+; SI-NEXT:    v_mul_f32_e64 v0, |s2|, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabsf_fn_fold:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_mul_f32_e64 v2, |s2|, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @fabsf(float %in0)
   %fmul = fmul float %fabs, %in1
   store float %fmul, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}fabs_fold:
-; SI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x9
-; VI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x24
-; GCN-NOT: and
-; GCN: v_mov_b32_e32 [[V_MUL_VI:v[0-9]+]], s[[#LOAD + 3]]
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, |s[[#LOAD + 2]]|, [[V_MUL_VI]]
 define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %in1) {
+; SI-LABEL: fabs_fold:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s3
+; SI-NEXT:    v_mul_f32_e64 v0, |s2|, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabs_fold:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_mul_f32_e64 v2, |s2|, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @llvm.fabs.f32(float %in0)
   %fmul = fmul float %fabs, %in1
   store float %fmul, ptr addrspace(1) %out
   ret void
 }
 
-; Make sure we turn some integer operations back into fabsf
-; FUNC-LABEL: {{^}}bitpreserve_fabsf_f32:
-; GCN: v_add_f32_e64 v{{[0-9]+}}, |s{{[0-9]+}}|, 1.0
 define amdgpu_kernel void @bitpreserve_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: bitpreserve_fabsf_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_add_f32_e64 v0, |s0|, 1.0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: bitpreserve_fabsf_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_add_f32_e64 v2, |s0|, 1.0
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %in.bc = bitcast float %in to i32
   %int.abs = and i32 %in.bc, 2147483647
   %bc = bitcast i32 %int.abs to float
@@ -111,3 +263,5 @@ declare float @fabsf(float) readnone
 declare float @llvm.fabs.f32(float) readnone
 declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone
 declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fail-select-buffer-atomic-fadd.ll b/llvm/test/CodeGen/AMDGPU/fail-select-buffer-atomic-fadd.ll
deleted file mode 100644
index 8ff78aaccf5a3ef7d85a5082fbc8c9bb0bf65b0c..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/fail-select-buffer-atomic-fadd.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=tahiti -o /dev/null %s 2>&1 | FileCheck -check-prefix=FAIL %s
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=hawaii -o /dev/null %s 2>&1 | FileCheck -check-prefix=FAIL %s
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=fiji -o /dev/null %s 2>&1 | FileCheck -check-prefix=FAIL %s
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=FAIL %s
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1010 -o /dev/null %s 2>&1 | FileCheck -check-prefix=FAIL %s
-
-; Make sure selection of these intrinsics fails on targets that do not
-; have the instruction available.
-; FIXME: Should also really make sure the v2f16 version fails.
-
-; FAIL: LLVM ERROR: Cannot select: {{.+}}: f32,ch = BUFFER_ATOMIC_FADD
-define amdgpu_cs void @atomic_fadd(<4 x i32> inreg %arg0) {
-  %ret = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float 1.0, <4 x i32> %arg0, i32 0, i32 112, i1 false)
-  ret void
-}
-
-declare float @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1 immarg) #0
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir b/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir
index 0fa6577b0dd03f8a7bacaa0c146ea283da18820f..01cd5f74dcb48f6d90f2f10a0523146fdd397cf4 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir
+++ b/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck %s
 
 # Make sure incorrect kills aren't emitted on vcc
 
diff --git a/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir b/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir
index ea03f2c57ea59039cc2a4d1e6cd43a7e6ac1f64c..8c969eb0ef9b664149868d71641e5ed569012c6b 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir
+++ b/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx902 -verify-machineinstrs -run-pass=regallocfast %s -o - | FileCheck --check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx902 -verify-machineinstrs -passes=regallocfast %s -o - | FileCheck --check-prefix=GCN %s
 
 ---
 name: fast_regalloc_bundle_handling
diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir
index 7642775aafdaf0b9a8197b7c228d4053c9660b43..6a2a2b0c3939fa828f7b72db329c953de54588c9 100644
--- a/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -passes=regallocfast -o - %s | FileCheck %s
 
 # This would hit "Illegal subregister index for physical register" verifier error since
 # tied operands would skip dropping the subregister index.
diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir
index 24b82f4862afdaddd3731a34f29276cf848a24e4..918242e9f99676adc08b66c5e962385708bf73e0 100644
--- a/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir
+++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name: self_loop_single_def_use
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
index 3ec36f03a48aa497b447d65c3e63ed14d1d1e84a..9ce1ba3316dd5b2a782ef9b99e0622c0d774ddf2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
@@ -497,47 +497,19 @@ define <2 x float> @v_fmaximum3_v2f32(<2 x float> %a, <2 x float> %b, <2 x float
 ; GFX9-LABEL: v_fmaximum3_v2f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v4, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v5, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b)
   %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %c, <2 x float> %max0)
@@ -559,47 +531,19 @@ define <2 x float> @v_fmaximum3_v2f32_commute(<2 x float> %a, <2 x float> %b, <2
 ; GFX9-LABEL: v_fmaximum3_v2f32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v0, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b)
   %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %max0, <2 x float> %c)
@@ -621,47 +565,19 @@ define <2 x float> @v_fmaximum3_v2f32__fabs_all(<2 x float> %a, <2 x float> %b,
 ; GFX9-LABEL: v_fmaximum3_v2f32__fabs_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, |v1|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e64 v6, |v1|, |v3|
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v1|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v7, |v6|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v1|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v6, |v1|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v3|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, |v2|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v0|, |v2|
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v0|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, |v0|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v2|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v2|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v0, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, |v4|, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v1|, |v3|
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, |v0|, |v2|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v0|, |v2|
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v2, v0, |v4|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v4|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v1, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, |v5|, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_max_f32_e64 v2, v1, |v5|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v5|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
   %b.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %b)
@@ -686,47 +602,19 @@ define <2 x float> @v_fmaximum3_v2f32__fneg_all(<2 x float> %a, <2 x float> %b,
 ; GFX9-LABEL: v_fmaximum3_v2f32__fneg_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, -v1, -v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e64 v6, -v1, -v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v1, -v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v7, -v6, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v6, -v1, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, -v0, -v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v0, -v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, -v0, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v2, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v0, -v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, -v4, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v1, -v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, -v0, -v2
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v0, -v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v2, v0, -v4
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, -v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v1, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, -v5, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_max_f32_e64 v2, v1, -v5
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg <2 x float> %a
   %b.fneg = fneg <2 x float> %b
@@ -751,35 +639,19 @@ define <2 x float> @v_fmaximum3_v2f32__inlineimm1(<2 x float> %a, <2 x float> %c
 ; GFX9-LABEL: v_fmaximum3_v2f32__inlineimm1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, 2.0, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, 2.0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, 2.0, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, 2.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> <float 2.0, float 2.0>)
   %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %max0, <2 x float> %c)
@@ -801,33 +673,17 @@ define <2 x float> @v_fmaximum3_v2f32__inlineimm2(<2 x float> %a, <2 x float> %b
 ; GFX9-LABEL: v_fmaximum3_v2f32__inlineimm2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 4.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, 4.0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, 4.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 4.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, 4.0, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, 4.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -852,67 +708,25 @@ define <3 x float> @v_fmaximum3_v3f32(<3 x float> %a, <3 x float> %b, <3 x float
 ; GFX9-LABEL: v_fmaximum3_v3f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v9, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_max_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v6, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v7, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v8, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> %b)
   %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %c, <3 x float> %max0)
@@ -935,67 +749,25 @@ define <3 x float> @v_fmaximum3_v3f32_commute(<3 x float> %a, <3 x float> %b, <3
 ; GFX9-LABEL: v_fmaximum3_v3f32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v9, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_max_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v6, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v0, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v1, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v2, v8
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> %b)
   %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %max0, <3 x float> %c)
@@ -1018,67 +790,25 @@ define <3 x float> @v_fmaximum3_v3f32__fabs_all(<3 x float> %a, <3 x float> %b,
 ; GFX9-LABEL: v_fmaximum3_v3f32__fabs_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, |v2|, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e64 v9, |v2|, |v5|
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v2|, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, |v9|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v2|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v9, |v2|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v5|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, |v1|, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v1|, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v10, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v1|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, |v1|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v4|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v0|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v0|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, |v0|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v3|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v0, |v6|
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v6|, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v2|, |v5|
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_max_f32_e64 v5, |v1|, |v4|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v1|, |v4|
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_max_f32_e64 v4, |v0|, |v3|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v0|, |v3|
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v0, |v6|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, |v6|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v6|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v6|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v1, |v7|
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v7|, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v1, |v7|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, |v7|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v7|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v7|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v2, |v8|
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v8|, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v2, |v8|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v2, |v8|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v8|, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, |v8|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %a)
   %b.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %b)
@@ -1104,67 +834,25 @@ define <3 x float> @v_fmaximum3_v3f32__fneg_all(<3 x float> %a, <3 x float> %b,
 ; GFX9-LABEL: v_fmaximum3_v3f32__fneg_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, -v2, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e64 v9, -v2, -v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v2, -v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, -v9, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v9, -v2, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, -v1, -v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v1, -v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v10, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 vcc, -v0, -v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v0, -v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, -v0, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v0, -v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v6, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v2, -v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_max_f32_e64 v5, -v1, -v4
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v1, -v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_max_f32_e64 v4, -v0, -v3
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v0, -v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v0, -v6
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, -v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v6, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v1, -v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v7, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v1, -v7
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, -v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v7, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_gt_f32_e64 s[4:5], v2, -v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v8, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_max_f32_e64 v3, v2, -v8
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v2, -v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -v8, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg <3 x float> %a
   %b.fneg = fneg <3 x float> %b
@@ -1190,49 +878,25 @@ define <3 x float> @v_fmaximum3_v3f32__inlineimm1(<3 x float> %a, <3 x float> %c
 ; GFX9-LABEL: v_fmaximum3_v3f32__inlineimm1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, 2.0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, 2.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, 2.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> <float 2.0, float 2.0, float 2.0>)
   %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %max0, <3 x float> %c)
@@ -1255,47 +919,23 @@ define <3 x float> @v_fmaximum3_v3f32__inlineimm2(<3 x float> %a, <3 x float> %b
 ; GFX9-LABEL: v_fmaximum3_v3f32__inlineimm2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 4.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, 4.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 4.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, 4.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, 4.0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, 4.0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
index 0e0b73b88d2dca67892e420e69dcb975d05eb6f6..21074d58bdb7e28db41a40dd0f0125f8af3961be 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
@@ -497,47 +497,19 @@ define <2 x float> @v_fminimum3_v2f32(<2 x float> %a, <2 x float> %b, <2 x float
 ; GFX9-LABEL: v_fminimum3_v2f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v4, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v5, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b)
   %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %c, <2 x float> %max0)
@@ -559,47 +531,19 @@ define <2 x float> @v_fminimum3_v2f32_commute(<2 x float> %a, <2 x float> %b, <2
 ; GFX9-LABEL: v_fminimum3_v2f32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v0, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b)
   %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %max0, <2 x float> %c)
@@ -621,47 +565,19 @@ define <2 x float> @v_fminimum3_v2f32__fabs_all(<2 x float> %a, <2 x float> %b,
 ; GFX9-LABEL: v_fminimum3_v2f32__fabs_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v1|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e64 v6, |v1|, |v3|
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v1|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v7, |v6|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v1|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v6, |v1|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v3|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, |v2|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v0|, |v2|
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v0|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, |v0|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v2|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v2|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v0, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, |v4|, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v1|, |v3|
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, |v0|, |v2|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v0|, |v2|
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v2, v0, |v4|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v4|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v1, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, |v5|, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_min_f32_e64 v2, v1, |v5|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v5|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
   %b.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %b)
@@ -686,47 +602,19 @@ define <2 x float> @v_fminimum3_v2f32__fneg_all(<2 x float> %a, <2 x float> %b,
 ; GFX9-LABEL: v_fminimum3_v2f32__fneg_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, -v1, -v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e64 v6, -v1, -v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v1, -v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v7, -v6, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v6, -v1, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, -v0, -v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v0, -v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, -v0, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v2, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v0, -v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, -v4, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v1, -v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, -v0, -v2
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v0, -v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v2, v0, -v4
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, -v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v1, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, -v5, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX9-NEXT:    v_min_f32_e64 v2, v1, -v5
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg <2 x float> %a
   %b.fneg = fneg <2 x float> %b
@@ -751,35 +639,19 @@ define <2 x float> @v_fminimum3_v2f32__inlineimm1(<2 x float> %a, <2 x float> %c
 ; GFX9-LABEL: v_fminimum3_v2f32__inlineimm1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 2.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, 2.0, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, 2.0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 2.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, 2.0, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, 2.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> <float 2.0, float 2.0>)
   %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %max0, <2 x float> %c)
@@ -801,33 +673,17 @@ define <2 x float> @v_fminimum3_v2f32__inlineimm2(<2 x float> %a, <2 x float> %b
 ; GFX9-LABEL: v_fminimum3_v2f32__inlineimm2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 4.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, 4.0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, 4.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 4.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, 4.0, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, 4.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -852,67 +708,25 @@ define <3 x float> @v_fminimum3_v3f32(<3 x float> %a, <3 x float> %b, <3 x float
 ; GFX9-LABEL: v_fminimum3_v3f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v9, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_min_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v6, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v7, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v8, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> %b)
   %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %c, <3 x float> %max0)
@@ -935,67 +749,25 @@ define <3 x float> @v_fminimum3_v3f32_commute(<3 x float> %a, <3 x float> %b, <3
 ; GFX9-LABEL: v_fminimum3_v3f32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v9, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_min_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v6, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v0, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v1, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v2, v8
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> %b)
   %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %max0, <3 x float> %c)
@@ -1018,67 +790,25 @@ define <3 x float> @v_fminimum3_v3f32__fabs_all(<3 x float> %a, <3 x float> %b,
 ; GFX9-LABEL: v_fminimum3_v3f32__fabs_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v2|, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e64 v9, |v2|, |v5|
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v2|, |v5|
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, |v9|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v2|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v9, |v2|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v5|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v1|, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v1|, |v4|
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v10, |v5|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v1|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, |v1|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v4|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], |v0|, |v3|
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, |v4|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v0|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, |v0|, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v3|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v3|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v0, |v6|
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v6|, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v2|, |v5|
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_min_f32_e64 v5, |v1|, |v4|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v1|, |v4|
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_min_f32_e64 v4, |v0|, |v3|
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, |v0|, |v3|
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v0, |v6|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, |v6|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v6|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, |v6|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v1, |v7|
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v7|, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v1, |v7|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, |v7|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v7|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, |v7|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v2, |v8|
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, |v8|, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v2, |v8|
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v2, |v8|
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], |v8|, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, |v8|, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %a)
   %b.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %b)
@@ -1104,67 +834,25 @@ define <3 x float> @v_fminimum3_v3f32__fneg_all(<3 x float> %a, <3 x float> %b,
 ; GFX9-LABEL: v_fminimum3_v3f32__fneg_all:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, -v2, -v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e64 v9, -v2, -v5
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v2, -v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, -v9, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v9, -v2, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, -v1, -v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v1, -v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v10, -v5, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, -v0, -v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], -v0, -v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, -v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, -v0, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v3, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v0, -v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v6, v0, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v2, -v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v9, vcc
+; GFX9-NEXT:    v_min_f32_e64 v5, -v1, -v4
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v1, -v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v5, vcc
+; GFX9-NEXT:    v_min_f32_e64 v4, -v0, -v3
+; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, -v0, -v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v4, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v0, -v6
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v0, -v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -v6, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v1, -v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v7, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v1, -v7
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v1, -v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -v7, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cmp_lt_f32_e64 s[4:5], v2, -v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, -v8, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GFX9-NEXT:    v_min_f32_e64 v3, v2, -v8
 ; GFX9-NEXT:    v_cmp_o_f32_e64 vcc, v2, -v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 s[4:5], -v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -v8, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg <3 x float> %a
   %b.fneg = fneg <3 x float> %b
@@ -1190,49 +878,25 @@ define <3 x float> @v_fminimum3_v3f32__inlineimm1(<3 x float> %a, <3 x float> %c
 ; GFX9-LABEL: v_fminimum3_v3f32__inlineimm1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 2.0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, 2.0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 2.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, 2.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 2.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, 2.0, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, 2.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> <float 2.0, float 2.0, float 2.0>)
   %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %max0, <3 x float> %c)
@@ -1255,47 +919,23 @@ define <3 x float> @v_fminimum3_v3f32__inlineimm2(<3 x float> %a, <3 x float> %b
 ; GFX9-LABEL: v_fminimum3_v3f32__inlineimm2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v2, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e32 v5, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 4.0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, 4.0, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 4.0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, 4.0, v1
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, 4.0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 4.0, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, 4.0, v2
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4f5271ed23252a1c790996d36602591341fe60d2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=R600 %s
+
+define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x, float %y) {
+; R600-LABEL: fneg_fabsf_fadd_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     ADD * T1.X, KC0[2].W, -|KC0[2].Z|,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %fsub = fsub float -0.000000e+00, %fabs
+  %fadd = fadd float %y, %fsub
+  store float %fadd, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x, float %y) {
+; R600-LABEL: fneg_fabsf_fmul_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MUL_IEEE * T1.X, KC0[2].W, -|KC0[2].Z|,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %fsub = fsub float -0.000000e+00, %fabs
+  %fmul = fmul float %y, %fsub
+  store float %fmul, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
+; R600-LABEL: fneg_fabsf_free_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T0.X, -PV.W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %bc = bitcast i32 %in to float
+  %fabs = call float @llvm.fabs.f32(float %bc)
+  %fsub = fsub float -0.000000e+00, %fabs
+  store float %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in) {
+; R600-LABEL: fneg_fabsf_fn_free_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T0.X, -PV.W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %bc = bitcast i32 %in to float
+  %fabs = call float @fabsf(float %bc)
+  %fsub = fsub float -0.000000e+00, %fabs
+  store float %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; R600-LABEL: fneg_fabsf_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[2].Z,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T0.X, -PV.W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call float @llvm.fabs.f32(float %in)
+  %fsub = fsub float -0.000000e+00, %fabs
+  store float %fsub, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; R600-LABEL: v_fneg_fabsf_f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @6
+; R600-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 8:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 9:
+; R600-NEXT:     MOV * T0.W, |T0.X|,
+; R600-NEXT:     MOV T0.X, -PV.W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %val = load float, ptr addrspace(1) %in, align 4
+  %fabs = call float @llvm.fabs.f32(float %val)
+  %fsub = fsub float -0.000000e+00, %fabs
+  store float %fsub, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
+; R600-LABEL: fneg_fabsf_v2f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 7, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV T0.W, KC0[3].X,
+; R600-NEXT:     MOV * T1.W, KC0[2].W,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T0.Y, -PV.W,
+; R600-NEXT:     MOV * T0.W, |T1.W|,
+; R600-NEXT:     MOV T0.X, -PV.W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
+  %fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs
+  store <2 x float> %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
+; R600-LABEL: fneg_fabsf_v4f32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 13, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.W, KC0[4].X,
+; R600-NEXT:     MOV T0.W, |PV.W|,
+; R600-NEXT:     MOV * T1.W, KC0[3].W,
+; R600-NEXT:     MOV T0.Z, KC0[3].Z,
+; R600-NEXT:     MOV T1.W, |PS|,
+; R600-NEXT:     MOV * T2.W, -PV.W,
+; R600-NEXT:     MOV T2.Z, -PV.W,
+; R600-NEXT:     MOV T0.W, KC0[3].Y,
+; R600-NEXT:     MOV * T1.W, |PV.Z|,
+; R600-NEXT:     MOV T2.Y, -PS,
+; R600-NEXT:     MOV * T0.W, |PV.W|,
+; R600-NEXT:     MOV T2.X, -PV.W,
+; R600-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
+  %fsub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %fabs
+  store <4 x float> %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+declare float @fabsf(float) readnone
+declare float @llvm.fabs.f32(float) readnone
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll
index 5f1d232daabe52c1d16c68902917390dc8c1ff58..2c9042ec17da88b6547751fc1b96dd54d674c66c 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll
@@ -1,12 +1,35 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefixes=SI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=VI,GCN %s
 
-; FIXME: Check something here. Currently it seems fabs + fneg aren't
-; into 2 modifiers, although theoretically that should work.
-
-; GCN-LABEL: {{^}}fneg_fabs_fadd_f64:
-; GCN: v_add_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}|
 define amdgpu_kernel void @fneg_fabs_fadd_f64(ptr addrspace(1) %out, double %x, double %y) {
+; SI-LABEL: fneg_fabs_fadd_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_add_f64 v[0:1], s[8:9], -|v[0:1]|
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_fadd_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_f64 v[0:1], s[0:1], -|v[0:1]|
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %fabs = call double @llvm.fabs.f64(double %x)
   %fsub = fsub double -0.000000e+00, %fabs
   %fadd = fadd double %y, %fsub
@@ -15,6 +38,29 @@ define amdgpu_kernel void @fneg_fabs_fadd_f64(ptr addrspace(1) %out, double %x,
 }
 
 define amdgpu_kernel void @v_fneg_fabs_fadd_f64(ptr addrspace(1) %out, ptr addrspace(1) %xptr, ptr addrspace(1) %yptr) {
+; SI-LABEL: v_fneg_fabs_fadd_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_add_f64 v[0:1], s[4:5], -|s[4:5]|
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fneg_fabs_fadd_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_f64 v[0:1], s[2:3], -|s[2:3]|
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %x = load double, ptr addrspace(1) %xptr, align 8
   %y = load double, ptr addrspace(1) %xptr, align 8
   %fabs = call double @llvm.fabs.f64(double %x)
@@ -24,9 +70,34 @@ define amdgpu_kernel void @v_fneg_fabs_fadd_f64(ptr addrspace(1) %out, ptr addrs
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_fmul_f64:
-; GCN: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}|
 define amdgpu_kernel void @fneg_fabs_fmul_f64(ptr addrspace(1) %out, double %x, double %y) {
+; SI-LABEL: fneg_fabs_fmul_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_mul_f64 v[0:1], s[8:9], -|v[0:1]|
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_fmul_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mul_f64 v[0:1], s[0:1], -|v[0:1]|
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %fabs = call double @llvm.fabs.f64(double %x)
   %fsub = fsub double -0.000000e+00, %fabs
   %fmul = fmul double %y, %fsub
@@ -34,8 +105,32 @@ define amdgpu_kernel void @fneg_fabs_fmul_f64(ptr addrspace(1) %out, double %x,
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_free_f64:
 define amdgpu_kernel void @fneg_fabs_free_f64(ptr addrspace(1) %out, i64 %in) {
+; SI-LABEL: fneg_fabs_free_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s3, 31
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_free_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_or_b32 s0, s3, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %bc = bitcast i64 %in to double
   %fabs = call double @llvm.fabs.f64(double %bc)
   %fsub = fsub double -0.000000e+00, %fabs
@@ -43,10 +138,32 @@ define amdgpu_kernel void @fneg_fabs_free_f64(ptr addrspace(1) %out, i64 %in) {
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_fn_free_f64:
-; SI: s_bitset1_b32
-; VI:  s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000
 define amdgpu_kernel void @fneg_fabs_fn_free_f64(ptr addrspace(1) %out, i64 %in) {
+; SI-LABEL: fneg_fabs_fn_free_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s3, 31
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_fn_free_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_or_b32 s0, s3, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %bc = bitcast i64 %in to double
   %fabs = call double @fabs(double %bc)
   %fsub = fsub double -0.000000e+00, %fabs
@@ -54,38 +171,126 @@ define amdgpu_kernel void @fneg_fabs_fn_free_f64(ptr addrspace(1) %out, i64 %in)
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_f64:
-; SI-DAG: s_load_dwordx2 s[[[LO_X:[0-9]+]]:[[HI_X:[0-9]+]]], s[{{[0-9]+:[0-9]+}}], 0x13
-; VI-DAG: s_load_dwordx2 s[[[LO_X:[0-9]+]]:[[HI_X:[0-9]+]]], s[{{[0-9]+:[0-9]+}}], 0x4c
-; GCN-DAG: s_bitset1_b32 s[[HI_X]], 31
-; GCN-DAG: v_mov_b32_e32 v[[LO_V:[0-9]+]], s[[LO_X]]
-; GCN-DAG: v_mov_b32_e32 v[[HI_V:[0-9]+]], s[[HI_X]]
-; GCN: buffer_store_dwordx2 v[[[LO_V]]:[[HI_V]]]
 define amdgpu_kernel void @fneg_fabs_f64(ptr addrspace(1) %out, [8 x i32], double %in) {
+; SI-LABEL: fneg_fabs_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s5, 31
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s3, 31
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %fabs = call double @llvm.fabs.f64(double %in)
   %fsub = fsub double -0.000000e+00, %fabs
   store double %fsub, ptr addrspace(1) %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_v2f64:
-; GCN-NOT: 0x80000000
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @fneg_fabs_v2f64(ptr addrspace(1) %out, <2 x double> %in) {
+; SI-LABEL: fneg_fabs_v2f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s7, 31
+; SI-NEXT:    s_bitset1_b32 s5, 31
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_v2f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_or_b32 s2, s7, 0x80000000
+; VI-NEXT:    s_or_b32 s3, s5, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v3, s2
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
   %fabs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %in)
   %fsub = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %fabs
   store <2 x double> %fsub, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}fneg_fabs_v4f64:
-; GCN-NOT: 0x80000000
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
-; GCN: s_bitset1_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @fneg_fabs_v4f64(ptr addrspace(1) %out, <4 x double> %in) {
+; SI-LABEL: fneg_fabs_v4f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x11
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s7, 31
+; SI-NEXT:    s_bitset1_b32 s11, 31
+; SI-NEXT:    s_bitset1_b32 s9, 31
+; SI-NEXT:    s_bitset1_b32 s5, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_mov_b32_e32 v4, s4
+; SI-NEXT:    v_mov_b32_e32 v6, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v3, s11
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; SI-NEXT:    v_mov_b32_e32 v5, s5
+; SI-NEXT:    v_mov_b32_e32 v7, s7
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_v4f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x44
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s7, 31
+; VI-NEXT:    s_bitset1_b32 s5, 31
+; VI-NEXT:    s_or_b32 s2, s11, 0x80000000
+; VI-NEXT:    s_or_b32 s3, s9, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v3, s2
+; VI-NEXT:    s_add_u32 s2, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_addc_u32 s3, s1, 0
+; VI-NEXT:    v_mov_b32_e32 v5, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s8
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_mov_b32_e32 v4, s2
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
   %fabs = call <4 x double> @llvm.fabs.v4f64(<4 x double> %in)
   %fsub = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %fabs
   store <4 x double> %fsub, ptr addrspace(1) %out
@@ -96,3 +301,5 @@ declare double @fabs(double) readnone
 declare double @llvm.fabs.f64(double) readnone
 declare <2 x double> @llvm.fabs.v2f64(<2 x double>) readnone
 declare <4 x double> @llvm.fabs.v4f64(<4 x double>) readnone
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
index b0c17828cb13bd3fe674834f5f0be006ea9b88d6..3c000d4fa63a38a02dc3f5330c700c27c44f2751 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
@@ -1,11 +1,31 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefixes=SI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck --check-prefixes=VI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=R600,FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn < %s | FileCheck --check-prefixes=SI,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefixes=VI,FUNC %s
 
-; FUNC-LABEL: {{^}}fneg_fabsf_fadd_f32:
-; SI-NOT: and
-; SI: v_sub_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, |{{v[0-9]+}}|
 define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x, float %y) {
+; SI-LABEL: fneg_fabsf_fadd_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_sub_f32_e64 v0, s3, |v0|
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_fadd_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_sub_f32_e64 v2, s3, |v0|
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @llvm.fabs.f32(float %x)
   %fsub = fsub float -0.000000e+00, %fabs
   %fadd = fadd float %y, %fsub
@@ -13,11 +33,30 @@ define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x,
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fabsf_fmul_f32:
-; SI-NOT: and
-; SI: v_mul_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, -|{{v[0-9]+}}|
-; SI-NOT: and
 define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x, float %y) {
+; SI-LABEL: fneg_fabsf_fmul_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mul_f32_e64 v0, s3, -|v0|
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_fmul_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mul_f32_e64 v2, s3, -|v0|
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @llvm.fabs.f32(float %x)
   %fsub = fsub float -0.000000e+00, %fabs
   %fmul = fmul float %y, %fsub
@@ -25,18 +64,30 @@ define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x,
   ret void
 }
 
-; DAGCombiner will transform:
-; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF))
-; unless isFabsFree returns true
-
-; FUNC-LABEL: {{^}}fneg_fabsf_free_f32:
-; R600-NOT: AND
-; R600: |PV.{{[XYZW]}}|
-; R600: -PV
-
-; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000
-; VI: s_bitset1_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: fneg_fabsf_free_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_or_b32 s4, s2, 0x80000000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_free_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %bc = bitcast i32 %in to float
   %fabs = call float @llvm.fabs.f32(float %bc)
   %fsub = fsub float -0.000000e+00, %fabs
@@ -44,13 +95,30 @@ define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fabsf_fn_free_f32:
-; R600-NOT: AND
-; R600: |PV.{{[XYZW]}}|
-; R600: -PV
-
-; SI: s_load_dwordx2 s[0:1], s[2:3], 0x9
 define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: fneg_fabsf_fn_free_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[2:3], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s4, 31
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_fn_free_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s4, s[2:3], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_or_b32 s2, s4, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %bc = bitcast i32 %in to float
   %fabs = call float @fabsf(float %bc)
   %fsub = fsub float -0.000000e+00, %fabs
@@ -58,18 +126,68 @@ define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fabsf_f32:
-; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000
 define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: fneg_fabsf_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_or_b32 s4, s2, 0x80000000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %fabs = call float @llvm.fabs.f32(float %in)
   %fsub = fsub float -0.000000e+00, %fabs
   store float %fsub, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_fabsf_f32:
-; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: v_fneg_fabsf_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, 0x80000000, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fneg_fabsf_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dword v2, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v2, 0x80000000, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %val = load float, ptr addrspace(1) %in, align 4
   %fabs = call float @llvm.fabs.f32(float %val)
   %fsub = fsub float -0.000000e+00, %fabs
@@ -77,28 +195,76 @@ define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fabsf_v2f32:
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: -PV
-; R600: |{{(PV|T[0-9])\.[XYZW]}}|
-; R600: -PV
-
-; FIXME: In this case two uses of the constant should be folded
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
+; SI-LABEL: fneg_fabsf_v2f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s3, 31
+; SI-NEXT:    s_bitset1_b32 s2, 31
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_v2f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s3, 31
+; VI-NEXT:    s_bitset1_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   %fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs
   store <2 x float> %fsub, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fabsf_v4f32:
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
-; SI: s_bitset1_b32 s{{[0-9]+}}, 31
 define amdgpu_kernel void @fneg_fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
+; SI-LABEL: fneg_fabsf_v4f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bitset1_b32 s7, 31
+; SI-NEXT:    s_bitset1_b32 s6, 31
+; SI-NEXT:    s_bitset1_b32 s5, 31
+; SI-NEXT:    s_bitset1_b32 s4, 31
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabsf_v4f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_or_b32 s2, s7, 0x80000000
+; VI-NEXT:    s_or_b32 s3, s6, 0x80000000
+; VI-NEXT:    s_bitset1_b32 s5, 31
+; VI-NEXT:    s_bitset1_b32 s4, 31
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v3, s2
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
   %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in)
   %fsub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %fabs
   store <4 x float> %fsub, ptr addrspace(1) %out
@@ -112,3 +278,5 @@ declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FUNC: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll
index 03ca780c903226f0cf7fd49b36c12fb4fef05b7a..d78bdfe08772a4b9b4c464fec7406b513cfbbc24 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.ll
@@ -1,89 +1,279 @@
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,VI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,GFX11 %s
-; RUN: not llc -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,R600 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s
+; RUN: not llc -mtriple=r600 -mcpu=redwood < %s
 
-; FUNC-LABEL: {{^}}s_fneg_f32:
-; R600: -PV
-
-; GCN: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GCN: s_xor_b32 [[NEG_VAL:s[0-9]+]], [[VAL]], 0x80000000
-; GCN: v_mov_b32_e32 v{{[0-9]+}}, [[NEG_VAL]]
 define amdgpu_kernel void @s_fneg_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: s_fneg_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = fsub float -0.000000e+00, %in
   store float %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_fneg_v2f32:
-; R600: -PV
-; R600: -PV
-
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
 define amdgpu_kernel void @s_fneg_v2f32(ptr addrspace(1) nocapture %out, <2 x float> %in) {
+; SI-LABEL: s_fneg_v2f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_xor_b32 s0, s3, 0x80000000
+; SI-NEXT:    s_xor_b32 s1, s2, 0x80000000
+; SI-NEXT:    v_mov_b32_e32 v0, s1
+; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_v2f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; VI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v2f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %in
   store <2 x float> %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_fneg_v4f32:
-; R600: -PV
-; R600: -T
-; R600: -PV
-; R600: -PV
-
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
-; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000
 define amdgpu_kernel void @s_fneg_v4f32(ptr addrspace(1) nocapture %out, <4 x float> %in) {
+; SI-LABEL: s_fneg_v4f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s7, s7, 0x80000000
+; SI-NEXT:    s_xor_b32 s6, s6, 0x80000000
+; SI-NEXT:    s_xor_b32 s5, s5, 0x80000000
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_v4f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s7, 0x80000000
+; VI-NEXT:    s_xor_b32 s3, s6, 0x80000000
+; VI-NEXT:    s_xor_b32 s5, s5, 0x80000000
+; VI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v3, s2
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v4f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x34
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s7, 0x80000000
+; GFX11-NEXT:    s_xor_b32 s3, s6, 0x80000000
+; GFX11-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; GFX11-NEXT:    s_xor_b32 s5, s5, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %in
   store <4 x float> %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; DAGCombiner will transform:
-; (fneg (f32 bitcast (i32 a))) => (f32 bitcast (xor (i32 a), 0x80000000))
-; unless the target returns true for isNegFree()
-
-; FUNC-LABEL: {{^}}fsub0_f32:
-
-; GCN: v_sub_f32_e64 v{{[0-9]}}, 0, s{{[0-9]+$}}
-
-; R600-NOT: XOR
-; R600: -KC0[2].Z
 define amdgpu_kernel void @fsub0_f32(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: fsub0_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_sub_f32_e64 v0, 0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fsub0_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_sub_f32_e64 v2, 0, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fsub0_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_sub_f32_e64 v1, 0, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %bc = bitcast i32 %in to float
   %fsub = fsub float 0.0, %bc
   store float %fsub, ptr addrspace(1) %out
   ret void
 }
-; FUNC-LABEL: {{^}}fneg_free_f32:
-; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
-; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
-; GFX11: s_load_b32 [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
-
-; GCN: s_xor_b32 [[RES:s[0-9]+]], [[NEG_VALUE]], 0x80000000
-; GCN: v_mov_b32_e32 [[V_RES:v[0-9]+]], [[RES]]
-; GCN: buffer_store_{{dword|b32}} [[V_RES]]
 
-; R600-NOT: XOR
-; R600: -PV.W
 define amdgpu_kernel void @fneg_free_f32(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: fneg_free_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_free_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fneg_free_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %bc = bitcast i32 %in to float
   %fsub = fsub float -0.0, %bc
   store float %fsub, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}fneg_fold_f32:
-; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
-; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
-; GFX11: s_load_{{dword|b32}} [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
-; GCN-NOT: xor
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, -[[NEG_VALUE]], [[NEG_VALUE]]
 define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: fneg_fold_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mul_f32_e64 v0, -s4, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fold_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mul_f32_e64 v2, -s2, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fneg_fold_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mul_f32_e64 v1, -s2, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fsub = fsub float -0.0, %in
   %fmul = fmul float %fsub, %in
   store float %fmul, ptr addrspace(1) %out
@@ -91,9 +281,41 @@ define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) {
 }
 
 ; Make sure we turn some integer operations back into fabs
-; FUNC-LABEL: {{^}}bitpreserve_fneg_f32:
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -4.0
 define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in) {
+; SI-LABEL: bitpreserve_fneg_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mul_f32_e64 v0, s4, -4.0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: bitpreserve_fneg_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mul_f32_e64 v2, s2, -4.0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: bitpreserve_fneg_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mul_f32_e64 v1, s2, -4.0
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %in.bc = bitcast float %in to i32
   %int.abs = xor i32 %in.bc, 2147483648
   %bc = bitcast i32 %int.abs to float
@@ -102,29 +324,94 @@ define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in
   ret void
 }
 
-; FUNC-LABEL: {{^}}s_fneg_i32:
-; GCN: s_load_{{dword|b32}} [[IN:s[0-9]+]]
-; GCN: s_xor_b32 [[FNEG:s[0-9]+]], [[IN]], 0x80000000
-; GCN: v_mov_b32_e32 [[V_FNEG:v[0-9]+]], [[FNEG]]
 define amdgpu_kernel void @s_fneg_i32(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: s_fneg_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = xor i32 %in, -2147483648
   store i32 %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i32:
-; GCN: s_waitcnt
-; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-NEXT: s_setpc_b64
 define i32 @v_fneg_i32(i32 %in) {
+; GCN-LABEL: v_fneg_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i32 %in, -2147483648
   ret i32 %fneg
 }
 
-; FUNC-LABEL: {{^}}s_fneg_i32_fp_use:
-; GCN: s_load_{{dword|b32}} [[IN:s[0-9]+]]
-; GCN: v_sub_f32_e64 v{{[0-9]+}}, 2.0, [[IN]]
 define amdgpu_kernel void @s_fneg_i32_fp_use(ptr addrspace(1) %out, i32 %in) {
+; SI-LABEL: s_fneg_i32_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_sub_f32_e64 v0, 2.0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_i32_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_sub_f32_e64 v2, 2.0, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_i32_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_sub_f32_e64 v1, 2.0, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = xor i32 %in, -2147483648
   %bitcast = bitcast i32 %fneg to float
   %fadd = fadd float %bitcast, 2.0
@@ -132,37 +419,105 @@ define amdgpu_kernel void @s_fneg_i32_fp_use(ptr addrspace(1) %out, i32 %in) {
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i32_fp_use:
-; GCN: s_waitcnt
-; GCN-NEXT: v_sub_f32_e32 v0, 2.0, v0
-; GCN-NEXT: s_setpc_b64
 define float @v_fneg_i32_fp_use(i32 %in) {
+; GCN-LABEL: v_fneg_i32_fp_use:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i32 %in, -2147483648
   %bitcast = bitcast i32 %fneg to float
   %fadd = fadd float %bitcast, 2.0
   ret float %fadd
 }
 
-; FUNC-LABEL: {{^}}s_fneg_i64:
-; GCN: s_xor_b32 s[[NEG_HI:[0-9]+]], s{{[0-9]+}}, 0x80000000
 define amdgpu_kernel void @s_fneg_i64(ptr addrspace(1) %out, i64 %in) {
+; SI-LABEL: s_fneg_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_xor_b32 s0, s3, 0x80000000
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_xor_b32 s0, s3, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = xor i64 %in, -9223372036854775808
   store i64 %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i64:
-; GCN: s_waitcnt
-; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-NEXT: s_setpc_b64
 define i64 @v_fneg_i64(i64 %in) {
+; GCN-LABEL: v_fneg_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i64 %in, -9223372036854775808
   ret i64 %fneg
 }
 
-; FUNC-LABEL: {{^}}s_fneg_i64_fp_use:
-; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, -s{{\[[0-9]+:[0-9]+\]}}, 2.0
 define amdgpu_kernel void @s_fneg_i64_fp_use(ptr addrspace(1) %out, i64 %in) {
+; SI-LABEL: s_fneg_i64_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_add_f64 v[0:1], -s[2:3], 2.0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_i64_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_f64 v[0:1], -s[2:3], 2.0
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_i64_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_f64 v[0:1], -s[2:3], 2.0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = xor i64 %in, -9223372036854775808
   %bitcast = bitcast i64 %fneg to double
   %fadd = fadd double %bitcast, 2.0
@@ -170,34 +525,65 @@ define amdgpu_kernel void @s_fneg_i64_fp_use(ptr addrspace(1) %out, i64 %in) {
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i64_fp_use:
-; GCN: s_waitcnt
-; GCN-NEXT: v_add_f64 v[0:1], -v[0:1], 2.0
-; GCN-NEXT: s_setpc_b64
 define double @v_fneg_i64_fp_use(i64 %in) {
+; GCN-LABEL: v_fneg_i64_fp_use:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], -v[0:1], 2.0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i64 %in, -9223372036854775808
   %bitcast = bitcast i64 %fneg to double
   %fadd = fadd double %bitcast, 2.0
   ret double %fadd
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i16:
-; GCN: s_waitcnt
-; GCN-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GCN-NEXT: s_setpc_b64
 define i16 @v_fneg_i16(i16 %in) {
+; GCN-LABEL: v_fneg_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i16 %in, -32768
   ret i16 %fneg
 }
 
-; FUNC-LABEL: {{^}}s_fneg_i16_fp_use:
-; SI: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], s{{[0-9]+}}
-; SI: v_sub_f32_e32 [[ADD:v[0-9]+]], 2.0, [[CVT0]]
-; SI: v_cvt_f16_f32_e32 [[CVT1:v[0-9]+]], [[ADD]]
-
-; VI: s_load_dword [[IN:s[0-9]+]]
-; VI: v_sub_f16_e64 v{{[0-9]+}}, 2.0, [[IN]]
 define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) {
+; SI-LABEL: s_fneg_i16_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, s2
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_i16_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_sub_f16_e64 v2, 2.0, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_i16_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_sub_f16_e64 v1, 2.0, s2
+; GFX11-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %fneg = xor i16 %in, -32768
   %bitcast = bitcast i16 %fneg to half
   %fadd = fadd half %bitcast, 2.0
@@ -205,69 +591,157 @@ define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) {
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_i16_fp_use:
-; SI: s_waitcnt
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0
-; SI-NEXT: s_setpc_b64
-
-; VI: s_waitcnt
-; VI-NEXT: v_sub_f16_e32 v0, 2.0, v0
-; VI-NEXT: s_setpc_b64
 define half @v_fneg_i16_fp_use(i16 %in) {
+; SI-LABEL: v_fneg_i16_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_fneg_i16_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_sub_f16_e32 v0, 2.0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fneg_i16_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_sub_f16_e32 v0, 2.0, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor i16 %in, -32768
   %bitcast = bitcast i16 %fneg to half
   %fadd = fadd half %bitcast, 2.0
   ret half %fadd
 }
 
-; FUNC-LABEL: {{^}}s_fneg_v2i16:
-; SI: s_xor_b32 s4, s4, 0x80008000
-
-; VI: s_lshr_b32 s5, s4, 16
-; VI: s_xor_b32 s4, s4, 0x8000
-; VI: s_xor_b32 s5, s5, 0x8000
-; VI: s_and_b32 s4, s4, 0xffff
-; VI: s_lshl_b32 s5, s5, 16
-; VI: s_or_b32 s4, s4, s5
 define amdgpu_kernel void @s_fneg_v2i16(ptr addrspace(1) %out, i32 %arg) {
+; SI-LABEL: s_fneg_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80008000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-NEXT:    s_xor_b32 s2, s2, 0x8000
+; VI-NEXT:    s_xor_b32 s3, s3, 0x8000
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_or_b32 s2, s2, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v2i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %in = bitcast i32 %arg to <2 x i16>
   %fneg = xor <2 x i16> %in, <i16 -32768, i16 -32768>
   store <2 x i16> %fneg, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_v2i16:
-; SI: v_xor_b32_e32 v1, 0x8000, v1
-; SI: v_xor_b32_e32 v0, 0x8000, v0
-; SI: v_lshlrev_b32_e32 v2, 16, v1
-; SI: v_and_b32_e32 v0, 0xffff, v0
-; SI: v_or_b32_e32 v0, v0, v2
-; SI: v_and_b32_e32 v1, 0xffff, v1
-
-; VI: s_waitcnt
-; VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; VI-NEXT: s_setpc_b64
 define <2 x i16> @v_fneg_v2i16(<2 x i16> %in) {
+; SI-LABEL: v_fneg_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; SI-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_fneg_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fneg_v2i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %fneg = xor <2 x i16> %in, <i16 -32768, i16 -32768>
   ret <2 x i16> %fneg
 }
 
-; FUNC-LABEL: {{^}}s_fneg_v2i16_fp_use:
-; SI: s_lshr_b32 s3, s2, 16
-; SI: v_cvt_f32_f16_e32 v0, s3
-; SI: v_cvt_f32_f16_e32 v1, s2
-; SI: v_sub_f32_e32 v0, 2.0, v0
-; SI: v_sub_f32_e32 v1, 2.0, v1
-
-; VI: s_lshr_b32 s5, s4, 16
-; VI: s_xor_b32 s5, s5, 0x8000
-; VI: s_xor_b32 s4, s4, 0x8000
-; VI: v_mov_b32_e32 v0, s5
-; VI: v_add_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI: v_add_f16_e64 v1, s4, 2.0
-; VI: v_or_b32_e32 v0, v1, v0
 define amdgpu_kernel void @s_fneg_v2i16_fp_use(ptr addrspace(1) %out, i32 %arg) {
+; SI-LABEL: s_fneg_v2i16_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_lshr_b32 s3, s2, 16
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, s3
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, s2
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_sub_f32_e32 v1, 2.0, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_v2i16_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0x4000
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-NEXT:    s_xor_b32 s3, s3, 0x8000
+; VI-NEXT:    s_xor_b32 s2, s2, 0x8000
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_f16_e64 v1, s2, 2.0
+; VI-NEXT:    v_add_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v1, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v2i16_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_pk_add_f16 v1, s2, 2.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %in = bitcast i32 %arg to <2 x i16>
   %fneg = xor <2 x i16> %in, <i16 -32768, i16 -32768>
   %bitcast = bitcast <2 x i16> %fneg to <2 x half>
@@ -276,20 +750,31 @@ define amdgpu_kernel void @s_fneg_v2i16_fp_use(ptr addrspace(1) %out, i32 %arg)
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_fneg_v2i16_fp_use:
-; SI: v_lshrrev_b32_e32 v1, 16, v0
-; SI: v_cvt_f32_f16_e32 v0, v0
-; SI: v_cvt_f32_f16_e32 v1, v1
-; SI: v_sub_f32_e32 v0, 2.0, v0
-; SI: v_sub_f32_e32 v1, 2.0, v1
-
-; VI: s_waitcnt
-; VI: v_mov_b32_e32 v1, 0x4000
-; VI: v_sub_f16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI: v_sub_f16_e32 v0, 2.0, v0
-; VI: v_or_b32_e32 v0, v0, v1
-; VI: s_setpc_b64
 define <2 x half> @v_fneg_v2i16_fp_use(i32 %arg) {
+; SI-LABEL: v_fneg_v2i16_fp_use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; SI-NEXT:    v_sub_f32_e32 v1, 2.0, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_fneg_v2i16_fp_use:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, 0x4000
+; VI-NEXT:    v_sub_f16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_sub_f16_e32 v0, 2.0, v0
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fneg_v2i16_fp_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %in = bitcast i32 %arg to <2 x i16>
   %fneg = xor <2 x i16> %in, <i16 -32768, i16 -32768>
   %bitcast = bitcast <2 x i16> %fneg to <2 x half>
diff --git a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
index 7c5f6d5e33efe73de20ce93566361145a0e857f2..301299daaa61f4b221b5f52488272c5daeb4d67a 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -372,52 +372,6 @@ define <8 x half> @fdiv_pow2_8xhalf(<8 x i16> %i) {
 }
 
 define double @fmul_pow_shl_cnt(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    movq %rax, %xmm1
-; CHECK-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; CHECK-SSE-NEXT:    addsd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vmovq %rax, %xmm0
-; CHECK-AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; CHECK-AVX2-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -462,52 +416,6 @@ define double @fmul_pow_shl_cnt(i64 %cnt) nounwind {
 }
 
 define double @fmul_pow_shl_cnt2(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $2, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    movq %rax, %xmm1
-; CHECK-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; CHECK-SSE-NEXT:    addsd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $2, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vmovq %rax, %xmm0
-; CHECK-AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; CHECK-AVX2-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $2, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $2, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -552,51 +460,6 @@ define double @fmul_pow_shl_cnt2(i64 %cnt) nounwind {
 }
 
 define float @fmul_pow_select(i32 %cnt, i1 %c) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_select:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    andl $1, %esi
-; CHECK-SSE-NEXT:    movl $2, %eax
-; CHECK-SSE-NEXT:    subl %esi, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_select:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    andl $1, %esi
-; CHECK-AVX2-NEXT:    movl $2, %eax
-; CHECK-AVX2-NEXT:    subl %esi, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_select:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    andl $1, %esi
-; CHECK-NO-FASTFMA-NEXT:    movl $2, %eax
-; CHECK-NO-FASTFMA-NEXT:    subl %esi, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_select:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    andl $1, %esi
-; CHECK-FMA-NEXT:    movl $2, %eax
-; CHECK-FMA-NEXT:    subl %esi, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_select:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -640,55 +503,6 @@ define float @fmul_pow_select(i32 %cnt, i1 %c) nounwind {
 }
 
 define float @fmul_fly_pow_mul_min_pow2(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_fly_pow_mul_min_pow2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $8, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    cmpq $8192, %rax # imm = 0x2000
-; CHECK-SSE-NEXT:    movl $8192, %ecx # imm = 0x2000
-; CHECK-SSE-NEXT:    cmovbq %rax, %rcx
-; CHECK-SSE-NEXT:    cvtsi2ss %rcx, %xmm0
-; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_fly_pow_mul_min_pow2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $8, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    cmpq $8192, %rax # imm = 0x2000
-; CHECK-AVX2-NEXT:    movl $8192, %ecx # imm = 0x2000
-; CHECK-AVX2-NEXT:    cmovbq %rax, %rcx
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_fly_pow_mul_min_pow2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    cmpq $8192, %rax # imm = 0x2000
-; CHECK-NO-FASTFMA-NEXT:    movl $8192, %ecx # imm = 0x2000
-; CHECK-NO-FASTFMA-NEXT:    cmovbq %rax, %rcx
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_fly_pow_mul_min_pow2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $8, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    cmpq $8192, %rax # imm = 0x2000
-; CHECK-FMA-NEXT:    movl $8192, %ecx # imm = 0x2000
-; CHECK-FMA-NEXT:    cmovbq %rax, %rcx
-; CHECK-FMA-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_fly_pow_mul_min_pow2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -757,63 +571,6 @@ define float @fmul_fly_pow_mul_min_pow2(i64 %cnt) nounwind {
 }
 
 define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_mul_max_pow2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $2, %eax
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    movl $1, %edx
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %edx
-; CHECK-SSE-NEXT:    cmpw %ax, %dx
-; CHECK-SSE-NEXT:    cmovbel %eax, %edx
-; CHECK-SSE-NEXT:    movzwl %dx, %eax
-; CHECK-SSE-NEXT:    cvtsi2sd %eax, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_mul_max_pow2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $2, %eax
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    movl $1, %edx
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %edx
-; CHECK-AVX2-NEXT:    cmpw %ax, %dx
-; CHECK-AVX2-NEXT:    cmovbel %eax, %edx
-; CHECK-AVX2-NEXT:    movzwl %dx, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_mul_max_pow2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $2, %eax
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %edx
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %edx
-; CHECK-NO-FASTFMA-NEXT:    cmpw %ax, %dx
-; CHECK-NO-FASTFMA-NEXT:    cmovbel %eax, %edx
-; CHECK-NO-FASTFMA-NEXT:    movzwl %dx, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_mul_max_pow2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $2, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    movl $1, %ecx
-; CHECK-FMA-NEXT:    shlxl %edi, %ecx, %ecx
-; CHECK-FMA-NEXT:    cmpw %ax, %cx
-; CHECK-FMA-NEXT:    cmoval %ecx, %eax
-; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_mul_max_pow2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -852,48 +609,6 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
 }
 
 define double @fmul_pow_shl_cnt_fail_maybe_non_pow2(i64 %v, i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rsi, %rcx
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rdi
-; CHECK-SSE-NEXT:    movq %rdi, %xmm1
-; CHECK-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; CHECK-SSE-NEXT:    addsd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rsi, %rcx
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rdi
-; CHECK-AVX2-NEXT:    vmovq %rdi, %xmm0
-; CHECK-AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; CHECK-AVX2-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rsi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rdi
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    shlxq %rsi, %rdi, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -938,90 +653,6 @@ define double @fmul_pow_shl_cnt_fail_maybe_non_pow2(i64 %v, i64 %cnt) nounwind {
 }
 
 define <2 x float> @fmul_pow_shl_cnt_vec_fail_expensive_cast(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_fail_expensive_cast:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [2,2]
-; CHECK-SSE-NEXT:    movdqa %xmm3, %xmm1
-; CHECK-SSE-NEXT:    psllq %xmm2, %xmm1
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm3
-; CHECK-SSE-NEXT:    movq %xmm3, %rax
-; CHECK-SSE-NEXT:    testq %rax, %rax
-; CHECK-SSE-NEXT:    js .LBB6_1
-; CHECK-SSE-NEXT:  # %bb.2:
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    jmp .LBB6_3
-; CHECK-SSE-NEXT:  .LBB6_1:
-; CHECK-SSE-NEXT:    movq %rax, %rcx
-; CHECK-SSE-NEXT:    shrq %rcx
-; CHECK-SSE-NEXT:    andl $1, %eax
-; CHECK-SSE-NEXT:    orq %rcx, %rax
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    addss %xmm0, %xmm0
-; CHECK-SSE-NEXT:  .LBB6_3:
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; CHECK-SSE-NEXT:    movq %xmm1, %rax
-; CHECK-SSE-NEXT:    testq %rax, %rax
-; CHECK-SSE-NEXT:    js .LBB6_4
-; CHECK-SSE-NEXT:  # %bb.5:
-; CHECK-SSE-NEXT:    xorps %xmm1, %xmm1
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    jmp .LBB6_6
-; CHECK-SSE-NEXT:  .LBB6_4:
-; CHECK-SSE-NEXT:    movq %rax, %rcx
-; CHECK-SSE-NEXT:    shrq %rcx
-; CHECK-SSE-NEXT:    andl $1, %eax
-; CHECK-SSE-NEXT:    orq %rcx, %rax
-; CHECK-SSE-NEXT:    xorps %xmm1, %xmm1
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    addss %xmm1, %xmm1
-; CHECK-SSE-NEXT:  .LBB6_6:
-; CHECK-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; CHECK-SSE-NEXT:    mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec_fail_expensive_cast:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpsrlq $1, %xmm0, %xmm1
-; CHECK-AVX2-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
-; CHECK-AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vmovq %xmm1, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
-; CHECK-AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
-; CHECK-AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
-; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; CHECK-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; CHECK-AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.5E+1,1.5E+1,1.5E+1,1.5E+1]
-; CHECK-AVX2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec_fail_expensive_cast:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpextrq $1, %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vmovq %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.5E+1,1.5E+1,1.5E+1,1.5E+1]
-; CHECK-NO-FASTFMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec_fail_expensive_cast:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec_fail_expensive_cast:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1110,60 +741,6 @@ define <2 x float> @fmul_pow_shl_cnt_vec_fail_expensive_cast(<2 x i64> %cnt) nou
 }
 
 define <2 x double> @fmul_pow_shl_cnt_vec(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [2,2]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm2
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
-; CHECK-SSE-NEXT:    movapd {{.*#+}} xmm0 = [4294967295,4294967295]
-; CHECK-SSE-NEXT:    andpd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    orpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    psrlq $32, %xmm1
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    addpd %xmm0, %xmm1
-; CHECK-SSE-NEXT:    mulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2pd %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1228,64 +805,6 @@ define <2 x double> @fmul_pow_shl_cnt_vec(<2 x i64> %cnt) nounwind {
 }
 
 define <4 x float> @fmul_pow_shl_cnt_vec_preserve_fma(<4 x i32> %cnt, <4 x float> %add) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pslld $23, %xmm0
-; CHECK-SSE-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2,2,2,2]
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE-NEXT:    pmuludq %xmm2, %xmm0
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; CHECK-SSE-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
-; CHECK-SSE-NEXT:    pand %xmm0, %xmm2
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; CHECK-SSE-NEXT:    psrld $16, %xmm0
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    subps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    addps %xmm2, %xmm0
-; CHECK-SSE-NEXT:    mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    addps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,2,2,2]
-; CHECK-AVX2-NEXT:    vpsllvd %xmm0, %xmm2, %xmm0
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1258291200,1258291200,1258291200,1258291200]
-; CHECK-AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; CHECK-AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1392508928,1392508928,1392508928,1392508928]
-; CHECK-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
-; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
-; CHECK-AVX2-NEXT:    vsubps %xmm3, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddps %xmm0, %xmm2, %xmm0
-; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [5.0E+0,5.0E+0,5.0E+0,5.0E+0]
-; CHECK-AVX2-NEXT:    vmulps %xmm2, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,2,2,2]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvd %xmm0, %xmm2, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtudq2ps %zmm0, %zmm0
-; CHECK-NO-FASTFMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [5.0E+0,5.0E+0,5.0E+0,5.0E+0]
-; CHECK-NO-FASTFMA-NEXT:    vmulps %xmm2, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vzeroupper
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,2,2,2]
-; CHECK-FMA-NEXT:    vpsllvd %xmm0, %xmm2, %xmm0
-; CHECK-FMA-NEXT:    vcvtudq2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1356,60 +875,6 @@ define <4 x float> @fmul_pow_shl_cnt_vec_preserve_fma(<4 x i32> %cnt, <4 x float
 }
 
 define <2 x double> @fmul_pow_shl_cnt_vec_non_splat_todo(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [2,2]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm2
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
-; CHECK-SSE-NEXT:    movapd {{.*#+}} xmm0 = [4294967295,4294967295]
-; CHECK-SSE-NEXT:    andpd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    orpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    psrlq $32, %xmm1
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    addpd %xmm0, %xmm1
-; CHECK-SSE-NEXT:    mulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2,2]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2pd %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1476,60 +941,6 @@ define <2 x double> @fmul_pow_shl_cnt_vec_non_splat_todo(<2 x i64> %cnt) nounwin
 }
 
 define <2 x double> @fmul_pow_shl_cnt_vec_non_splat2_todo(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [2,1]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm2
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
-; CHECK-SSE-NEXT:    movapd {{.*#+}} xmm0 = [4294967295,4294967295]
-; CHECK-SSE-NEXT:    andpd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    orpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    psrlq $32, %xmm1
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    addpd %xmm0, %xmm1
-; CHECK-SSE-NEXT:    mulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,1]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,1]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,1]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2pd %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1594,158 +1005,6 @@ define <2 x double> @fmul_pow_shl_cnt_vec_non_splat2_todo(<2 x i64> %cnt) nounwi
 }
 
 define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_fail_to_large:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    subq $40, %rsp
-; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; CHECK-SSE-NEXT:    pslld $23, %xmm0
-; CHECK-SSE-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; CHECK-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-SSE-NEXT:    pextrw $1, %xmm0, %eax
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %eax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-SSE-NEXT:    pextrw $0, %xmm0, %eax
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %eax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm0
-; CHECK-SSE-NEXT:    addq $40, %rsp
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec_fail_to_large:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    subq $40, %rsp
-; CHECK-AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2,2,2,2]
-; CHECK-AVX2-NEXT:    vpsllvd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-AVX2-NEXT:    vpextrw $0, %xmm0, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-AVX2-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-AVX2-NEXT:    addq $40, %rsp
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_vec_fail_to_large:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; CHECK-NO-FASTFMA-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,2,0,0,0,0,0,0]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvd %ymm0, %ymm1, %ymm0
-; CHECK-NO-FASTFMA-NEXT:    vpmovdw %zmm0, %ymm1
-; CHECK-NO-FASTFMA-NEXT:    vpextrw $0, %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpextrw $1, %xmm1, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm1, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; CHECK-NO-FASTFMA-NEXT:    vmovaps {{.*#+}} xmm1 = [16,0,0,0]
-; CHECK-NO-FASTFMA-NEXT:    xorl %eax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm2
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm2, %eax
-; CHECK-NO-FASTFMA-NEXT:    vmovd %eax, %xmm2
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastw %xmm2, %xmm2
-; CHECK-NO-FASTFMA-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm2
-; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm2, %ymm0
-; CHECK-NO-FASTFMA-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.5E+1,1.5E+1,1.5E+1,1.5E+1,1.5E+1,1.5E+1,1.5E+1,1.5E+1]
-; CHECK-NO-FASTFMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %ymm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vzeroupper
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_vec_fail_to_large:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [2,2,2,2,2,2,2,2]
-; CHECK-FMA-NEXT:    vpsllvw %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vpextrw $7, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm1
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
-; CHECK-FMA-NEXT:    vmovd %xmm1, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
-; CHECK-FMA-NEXT:    vpextrw $6, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm2
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; CHECK-FMA-NEXT:    vmovd %xmm2, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; CHECK-FMA-NEXT:    vpextrw $5, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm3, %xmm3
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
-; CHECK-FMA-NEXT:    vmovd %xmm3, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm3
-; CHECK-FMA-NEXT:    vpextrw $4, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm4, %xmm4
-; CHECK-FMA-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm4, %xmm2
-; CHECK-FMA-NEXT:    vmovd %xmm2, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; CHECK-FMA-NEXT:    vpextrw $3, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm4
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
-; CHECK-FMA-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; CHECK-FMA-NEXT:    vmovd %xmm4, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm3
-; CHECK-FMA-NEXT:    vpextrw $2, %xmm0, %eax
-; CHECK-FMA-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm2
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; CHECK-FMA-NEXT:    vmovd %xmm2, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm2
-; CHECK-FMA-NEXT:    vpextrw $1, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm4
-; CHECK-FMA-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm4, %xmm3
-; CHECK-FMA-NEXT:    vmovd %xmm3, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm3
-; CHECK-FMA-NEXT:    vpextrw $0, %xmm0, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; CHECK-FMA-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-FMA-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %ymm0
-; CHECK-FMA-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %ymm0, %xmm0
-; CHECK-FMA-NEXT:    vzeroupper
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_vec_fail_to_large:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1789,52 +1048,6 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
 }
 
 define double @fmul_pow_shl_cnt_fail_maybe_bad_exp(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_fail_maybe_bad_exp:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    movq %rax, %xmm1
-; CHECK-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    movapd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; CHECK-SSE-NEXT:    addsd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_fail_maybe_bad_exp:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vmovq %rax, %xmm0
-; CHECK-AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; CHECK-AVX2-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_fail_maybe_bad_exp:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_fail_maybe_bad_exp:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_fail_maybe_bad_exp:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1883,47 +1096,6 @@ define double @fmul_pow_shl_cnt_fail_maybe_bad_exp(i64 %cnt) nounwind {
 }
 
 define double @fmul_pow_shl_cnt_safe(i16 %cnt) nounwind {
-; CHECK-SSE-LABEL: fmul_pow_shl_cnt_safe:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    movzwl %ax, %eax
-; CHECK-SSE-NEXT:    cvtsi2sd %eax, %xmm0
-; CHECK-SSE-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_safe:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    movzwl %ax, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fmul_pow_shl_cnt_safe:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    movzwl %ax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fmul_pow_shl_cnt_safe:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fmul_pow_shl_cnt_safe:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1964,66 +1136,6 @@ define double @fmul_pow_shl_cnt_safe(i16 %cnt) nounwind {
 }
 
 define <2 x double> @fdiv_pow_shl_cnt_vec(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_vec:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,1]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm2
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
-; CHECK-SSE-NEXT:    movapd {{.*#+}} xmm0 = [4294967295,4294967295]
-; CHECK-SSE-NEXT:    andpd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    orpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    psrlq $32, %xmm1
-; CHECK-SSE-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-SSE-NEXT:    addpd %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movapd {{.*#+}} xmm0 = [1.0E+0,1.0E+0]
-; CHECK-SSE-NEXT:    divpd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_vec:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vmovddup {{.*#+}} xmm1 = [1.0E+0,1.0E+0]
-; CHECK-AVX2-NEXT:    # xmm1 = mem[0,0]
-; CHECK-AVX2-NEXT:    vdivpd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_vec:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovddup {{.*#+}} xmm1 = [1.0E+0,1.0E+0]
-; CHECK-NO-FASTFMA-NEXT:    # xmm1 = mem[0,0]
-; CHECK-NO-FASTFMA-NEXT:    vdivpd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_vec:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2pd %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovddup {{.*#+}} xmm1 = [1.0E+0,1.0E+0]
-; CHECK-FMA-NEXT:    # xmm1 = mem[0,0]
-; CHECK-FMA-NEXT:    vdivpd %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_vec:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2065,94 +1177,6 @@ define <2 x double> @fdiv_pow_shl_cnt_vec(<2 x i64> %cnt) nounwind {
 }
 
 define <2 x float> @fdiv_pow_shl_cnt_vec_with_expensive_cast(<2 x i64> %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_vec_with_expensive_cast:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; CHECK-SSE-NEXT:    movdqa %xmm3, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm1, %xmm2
-; CHECK-SSE-NEXT:    psllq %xmm0, %xmm3
-; CHECK-SSE-NEXT:    movq %xmm3, %rax
-; CHECK-SSE-NEXT:    testq %rax, %rax
-; CHECK-SSE-NEXT:    js .LBB15_1
-; CHECK-SSE-NEXT:  # %bb.2:
-; CHECK-SSE-NEXT:    xorps %xmm1, %xmm1
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    jmp .LBB15_3
-; CHECK-SSE-NEXT:  .LBB15_1:
-; CHECK-SSE-NEXT:    movq %rax, %rcx
-; CHECK-SSE-NEXT:    shrq %rcx
-; CHECK-SSE-NEXT:    andl $1, %eax
-; CHECK-SSE-NEXT:    orq %rcx, %rax
-; CHECK-SSE-NEXT:    xorps %xmm1, %xmm1
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    addss %xmm1, %xmm1
-; CHECK-SSE-NEXT:  .LBB15_3:
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; CHECK-SSE-NEXT:    movq %xmm0, %rax
-; CHECK-SSE-NEXT:    testq %rax, %rax
-; CHECK-SSE-NEXT:    js .LBB15_4
-; CHECK-SSE-NEXT:  # %bb.5:
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    jmp .LBB15_6
-; CHECK-SSE-NEXT:  .LBB15_4:
-; CHECK-SSE-NEXT:    movq %rax, %rcx
-; CHECK-SSE-NEXT:    shrq %rcx
-; CHECK-SSE-NEXT:    andl $1, %eax
-; CHECK-SSE-NEXT:    orq %rcx, %rax
-; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    addss %xmm0, %xmm0
-; CHECK-SSE-NEXT:  .LBB15_6:
-; CHECK-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = <1.0E+0,1.0E+0,u,u>
-; CHECK-SSE-NEXT:    divps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_vec_with_expensive_cast:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; CHECK-AVX2-NEXT:    vpsrlq $1, %xmm0, %xmm2
-; CHECK-AVX2-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; CHECK-AVX2-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
-; CHECK-AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
-; CHECK-AVX2-NEXT:    vmovq %xmm1, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
-; CHECK-AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
-; CHECK-AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
-; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; CHECK-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; CHECK-AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-AVX2-NEXT:    vdivps %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_vec_with_expensive_cast:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpextrq $1, %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm1
-; CHECK-NO-FASTFMA-NEXT:    vmovq %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-NO-FASTFMA-NEXT:    vdivps %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_vec_with_expensive_cast:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [1,1]
-; CHECK-FMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtuqq2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-FMA-NEXT:    vdivps %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_vec_with_expensive_cast:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2187,65 +1211,6 @@ define <2 x float> @fdiv_pow_shl_cnt_vec_with_expensive_cast(<2 x i64> %cnt) nou
 }
 
 define float @fdiv_pow_shl_cnt_fail_maybe_z(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_fail_maybe_z:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $8, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    testq %rax, %rax
-; CHECK-SSE-NEXT:    js .LBB16_1
-; CHECK-SSE-NEXT:  # %bb.2:
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    jmp .LBB16_3
-; CHECK-SSE-NEXT:  .LBB16_1:
-; CHECK-SSE-NEXT:    shrq %rax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    addss %xmm1, %xmm1
-; CHECK-SSE-NEXT:  .LBB16_3:
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_fail_maybe_z:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $8, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    testq %rax, %rax
-; CHECK-AVX2-NEXT:    js .LBB16_1
-; CHECK-AVX2-NEXT:  # %bb.2:
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    jmp .LBB16_3
-; CHECK-AVX2-NEXT:  .LBB16_1:
-; CHECK-AVX2-NEXT:    shrq %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vaddss %xmm0, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:  .LBB16_3:
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_fail_maybe_z:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_maybe_z:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $8, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2ss %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_fail_maybe_z:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2337,47 +1302,6 @@ define float @fdiv_pow_shl_cnt_fail_maybe_z(i64 %cnt) nounwind {
 }
 
 define float @fdiv_pow_shl_cnt_fail_neg_int(i64 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_fail_neg_int:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    movl $8, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_fail_neg_int:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    movl $8, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_fail_neg_int:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_neg_int:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $8, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_fail_neg_int:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2483,51 +1407,6 @@ define float @fdiv_pow_shl_cnt_fail_neg_int(i64 %cnt) nounwind {
 }
 
 define float @fdiv_pow_shl_cnt(i64 %cnt_in) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movq %rdi, %rcx
-; CHECK-SSE-NEXT:    andb $31, %cl
-; CHECK-SSE-NEXT:    movl $8, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-SSE-NEXT:    shlq %cl, %rax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movq %rdi, %rcx
-; CHECK-AVX2-NEXT:    andb $31, %cl
-; CHECK-AVX2-NEXT:    movl $8, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movq %rdi, %rcx
-; CHECK-NO-FASTFMA-NEXT:    andb $31, %cl
-; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
-; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    andb $31, %dil
-; CHECK-FMA-NEXT:    movl $8, %eax
-; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2560,70 +1439,6 @@ define float @fdiv_pow_shl_cnt(i64 %cnt_in) nounwind {
 }
 
 define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pushq %rax
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movaps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    popq %rax
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    pushq %rax
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    popq %rax
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_fail_out_of_bounds:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2674,74 +1489,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 }
 
 define half @fdiv_pow_shl_cnt_in_bounds(i16 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_in_bounds:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pushq %rax
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    movzwl %ax, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %eax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movaps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    popq %rax
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_in_bounds:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    pushq %rax
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    movzwl %ax, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    popq %rax
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_in_bounds:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    movzwl %ax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_in_bounds:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_in_bounds:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2770,74 +1517,6 @@ define half @fdiv_pow_shl_cnt_in_bounds(i16 %cnt) nounwind {
 }
 
 define half @fdiv_pow_shl_cnt_in_bounds2(i16 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_in_bounds2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pushq %rax
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    movzwl %ax, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %eax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movaps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    popq %rax
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_in_bounds2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    pushq %rax
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    movzwl %ax, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    popq %rax
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_in_bounds2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    movzwl %ax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_in_bounds2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_in_bounds2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2866,74 +1545,6 @@ define half @fdiv_pow_shl_cnt_in_bounds2(i16 %cnt) nounwind {
 }
 
 define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    pushq %rax
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    movzwl %ax, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %eax, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm0, %xmm1
-; CHECK-SSE-NEXT:    movaps %xmm1, %xmm0
-; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    popq %rax
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    pushq %rax
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    movzwl %ax, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
-; CHECK-AVX2-NEXT:    popq %rax
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    movzwl %ax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-NO-FASTFMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovd %xmm0, %eax
-; CHECK-FMA-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt_fail_out_of_bound2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2980,47 +1591,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 }
 
 define double @fdiv_pow_shl_cnt32_to_dbl_okay(i32 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt32_to_dbl_okay:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    cvtsi2sd %rax, %xmm1
-; CHECK-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-SSE-NEXT:    divsd %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt32_to_dbl_okay:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-AVX2-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt32_to_dbl_okay:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NO-FASTFMA-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt32_to_dbl_okay:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2sd %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-FMA-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt32_to_dbl_okay:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3056,47 +1626,6 @@ define double @fdiv_pow_shl_cnt32_to_dbl_okay(i32 %cnt) nounwind {
 }
 
 define float @fdiv_pow_shl_cnt32_out_of_bounds2(i32 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt32_out_of_bounds2:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt32_out_of_bounds2:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt32_out_of_bounds2:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt32_out_of_bounds2:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt32_out_of_bounds2:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3164,47 +1693,6 @@ define float @fdiv_pow_shl_cnt32_out_of_bounds2(i32 %cnt) nounwind {
 }
 
 define float @fdiv_pow_shl_cnt32_okay(i32 %cnt) nounwind {
-; CHECK-SSE-LABEL: fdiv_pow_shl_cnt32_okay:
-; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    movl %edi, %ecx
-; CHECK-SSE-NEXT:    movl $1, %eax
-; CHECK-SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-SSE-NEXT:    shll %cl, %eax
-; CHECK-SSE-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE-NEXT:    divss %xmm1, %xmm0
-; CHECK-SSE-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: fdiv_pow_shl_cnt32_okay:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    movl %edi, %ecx
-; CHECK-AVX2-NEXT:    movl $1, %eax
-; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-AVX2-NEXT:    retq
-;
-; CHECK-NO-FASTFMA-LABEL: fdiv_pow_shl_cnt32_okay:
-; CHECK-NO-FASTFMA:       # %bb.0:
-; CHECK-NO-FASTFMA-NEXT:    movl %edi, %ecx
-; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
-; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NO-FASTFMA-NEXT:    retq
-;
-; CHECK-FMA-LABEL: fdiv_pow_shl_cnt32_okay:
-; CHECK-FMA:       # %bb.0:
-; CHECK-FMA-NEXT:    movl $1, %eax
-; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
-; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; CHECK-FMA-NEXT:    retq
 ; VI-LABEL: fdiv_pow_shl_cnt32_okay:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/force-store-sc0-sc1.ll b/llvm/test/CodeGen/AMDGPU/force-store-sc0-sc1.ll
deleted file mode 100644
index b35de032030043f96e82be3a77a9e1f4ad07d07d..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/force-store-sc0-sc1.ll
+++ /dev/null
@@ -1,141 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,FORCESC0SC1 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx941 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,FORCESC0SC1 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs -mattr=-forcestoresc1 < %s | FileCheck --check-prefixes=GCN,NOSC0SC1 %s
-
-define amdgpu_kernel void @store_global(ptr addrspace(1) %ptr) {
-; FORCESC0SC1-LABEL: store_global:
-; FORCESC0SC1:       ; %bb.0: ; %entry
-; FORCESC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v0, 0
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v1, 1.0
-; FORCESC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; FORCESC0SC1-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
-; FORCESC0SC1-NEXT:    s_endpgm
-;
-; NOSC0SC1-LABEL: store_global:
-; NOSC0SC1:       ; %bb.0: ; %entry
-; NOSC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v0, 0
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v1, 1.0
-; NOSC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; NOSC0SC1-NEXT:    global_store_dword v0, v1, s[0:1]
-; NOSC0SC1-NEXT:    s_endpgm
-entry:
-  store float 1.000000e+00, ptr addrspace(1) %ptr, align 4
-  ret void
-}
-
-define amdgpu_kernel void @store_flat(ptr addrspace(0) %ptr) {
-; FORCESC0SC1-LABEL: store_flat:
-; FORCESC0SC1:       ; %bb.0: ; %entry
-; FORCESC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v2, 1.0
-; FORCESC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; FORCESC0SC1-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; FORCESC0SC1-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
-; FORCESC0SC1-NEXT:    s_endpgm
-;
-; NOSC0SC1-LABEL: store_flat:
-; NOSC0SC1:       ; %bb.0: ; %entry
-; NOSC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v2, 1.0
-; NOSC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; NOSC0SC1-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
-; NOSC0SC1-NEXT:    flat_store_dword v[0:1], v2
-; NOSC0SC1-NEXT:    s_endpgm
-entry:
-  store float 1.000000e+00, ptr addrspace(0) %ptr, align 4
-  ret void
-}
-
-define amdgpu_kernel void @store_lds(ptr addrspace(3) %ptr) {
-; GCN-LABEL: store_lds:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v0, 1.0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    ds_write_b32 v1, v0
-; GCN-NEXT:    s_endpgm
-entry:
-  store float 1.000000e+00, ptr addrspace(3) %ptr, align 4
-  ret void
-}
-
-define amdgpu_kernel void @store_scratch(ptr addrspace(5) %ptr) {
-; FORCESC0SC1-LABEL: store_scratch:
-; FORCESC0SC1:       ; %bb.0: ; %entry
-; FORCESC0SC1-NEXT:    s_load_dword s0, s[0:1], 0x24
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v0, 1.0
-; FORCESC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; FORCESC0SC1-NEXT:    scratch_store_dword off, v0, s0 sc0 sc1
-; FORCESC0SC1-NEXT:    s_endpgm
-;
-; NOSC0SC1-LABEL: store_scratch:
-; NOSC0SC1:       ; %bb.0: ; %entry
-; NOSC0SC1-NEXT:    s_load_dword s0, s[0:1], 0x24
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v0, 1.0
-; NOSC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; NOSC0SC1-NEXT:    scratch_store_dword off, v0, s0
-; NOSC0SC1-NEXT:    s_endpgm
-entry:
-  store float 1.000000e+00, ptr addrspace(5) %ptr, align 4
-  ret void
-}
-
-define amdgpu_ps void @store_buffer(<4 x i32> inreg %rsrc, float %data, i32 %index) {
-; FORCESC0SC1-LABEL: store_buffer:
-; FORCESC0SC1:       ; %bb.0: ; %main_body
-; FORCESC0SC1-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 idxen sc0 sc1
-; FORCESC0SC1-NEXT:    s_endpgm
-;
-; NOSC0SC1-LABEL: store_buffer:
-; NOSC0SC1:       ; %bb.0: ; %main_body
-; NOSC0SC1-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 idxen
-; NOSC0SC1-NEXT:    s_endpgm
-main_body:
-  call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-define amdgpu_kernel void @store_global_atomic(ptr addrspace(1) %ptr) {
-; FORCESC0SC1-LABEL: store_global_atomic:
-; FORCESC0SC1:       ; %bb.0: ; %entry
-; FORCESC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v0, 0
-; FORCESC0SC1-NEXT:    v_mov_b32_e32 v1, 1.0
-; FORCESC0SC1-NEXT:    buffer_wbl2 sc1
-; FORCESC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; FORCESC0SC1-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
-; FORCESC0SC1-NEXT:    s_endpgm
-;
-; NOSC0SC1-LABEL: store_global_atomic:
-; NOSC0SC1:       ; %bb.0: ; %entry
-; NOSC0SC1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v0, 0
-; NOSC0SC1-NEXT:    v_mov_b32_e32 v1, 1.0
-; NOSC0SC1-NEXT:    buffer_wbl2 sc1
-; NOSC0SC1-NEXT:    s_waitcnt lgkmcnt(0)
-; NOSC0SC1-NEXT:    global_store_dword v0, v1, s[0:1] sc1
-; NOSC0SC1-NEXT:    s_endpgm
-entry:
-  store atomic float 1.000000e+00, ptr addrspace(1) %ptr syncscope("agent-one-as") seq_cst, align 4
-  ret void
-}
-
-define amdgpu_kernel void @store_global_atomic_system(ptr addrspace(1) %ptr) {
-; GCN-LABEL: store_global_atomic_system:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    v_mov_b32_e32 v1, 1.0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
-; GCN-NEXT:    s_endpgm
-  store atomic float 1.000000e+00, ptr addrspace(1) %ptr monotonic, align 4
-  ret void
-}
-
-
-declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index 121fab51024fdd7a558bda1344fcc3931de151c0..0835b43e96a6781cf33aec30d66accc9852a1c7f 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -2,7 +2,6 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs | FileCheck %s -check-prefix=GFX90A
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 -verify-machineinstrs | FileCheck %s -check-prefix=GFX940
 
-declare double @llvm.amdgcn.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i1)
 declare double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)
 declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)
 declare double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32 immarg)
@@ -24,89 +23,6 @@ declare double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %ptr, double %data)
 declare double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %ptr, double %data)
 declare double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) nocapture, double, i32, i32, i1)
 
-define amdgpu_kernel void @buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
-; GFX90A-LABEL: buffer_atomic_add_noret_f64:
-; GFX90A:       ; %bb.0: ; %main_body
-; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX90A-NEXT:    s_load_dword s8, s[0:1], 0x3c
-; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
-; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen
-; GFX90A-NEXT:    s_endpgm
-;
-; GFX940-LABEL: buffer_atomic_add_noret_f64:
-; GFX940:       ; %bb.0: ; %main_body
-; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
-; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v2, s8
-; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen
-; GFX940-NEXT:    s_endpgm
-main_body:
-  %ret = call double @llvm.amdgcn.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret void
-}
-
-define amdgpu_ps void @buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {
-; GFX90A-LABEL: buffer_atomic_add_rtn_f64:
-; GFX90A:       ; %bb.0: ; %main_body
-; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen glc
-; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
-; GFX90A-NEXT:    s_endpgm
-;
-; GFX940-LABEL: buffer_atomic_add_rtn_f64:
-; GFX940:       ; %bb.0: ; %main_body
-; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0
-; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1] sc0 sc1
-; GFX940-NEXT:    s_endpgm
-main_body:
-  %ret = call double @llvm.amdgcn.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  store double %ret, ptr undef
-  ret void
-}
-
-define amdgpu_kernel void @buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
-; GFX90A-LABEL: buffer_atomic_add_rtn_f64_off4_slc:
-; GFX90A:       ; %bb.0: ; %main_body
-; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX90A-NEXT:    s_load_dword s10, s[0:1], 0x3c
-; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
-; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen offset:4 glc slc
-; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
-; GFX90A-NEXT:    s_endpgm
-;
-; GFX940-LABEL: buffer_atomic_add_rtn_f64_off4_slc:
-; GFX940:       ; %bb.0: ; %main_body
-; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
-; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
-; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
-; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen offset:4 sc0 nt
-; GFX940-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9] sc0 sc1
-; GFX940-NEXT:    s_endpgm
-main_body:
-  %ret = call double @llvm.amdgcn.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  store double %ret, ptr addrspace(1) %out, align 8
-  ret void
-}
-
 define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
 ; GFX90A-LABEL: raw_buffer_atomic_add_noret_f64:
 ; GFX90A:       ; %bb.0: ; %main_body
@@ -1186,7 +1102,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB42_3
+; GFX90A-NEXT:    s_cbranch_execz .LBB39_3
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
@@ -1198,7 +1114,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB42_2: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB39_2: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX90A-NEXT:    buffer_wbl2
@@ -1210,8 +1126,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB42_2
-; GFX90A-NEXT:  .LBB42_3:
+; GFX90A-NEXT:    s_cbranch_execnz .LBB39_2
+; GFX90A-NEXT:  .LBB39_3:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat:
@@ -1221,7 +1137,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB42_2
+; GFX940-NEXT:    s_cbranch_execz .LBB39_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1233,7 +1149,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    buffer_inv sc0 sc1
-; GFX940-NEXT:  .LBB42_2:
+; GFX940-NEXT:  .LBB39_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst
@@ -1248,7 +1164,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB43_2
+; GFX90A-NEXT:    s_cbranch_execz .LBB40_2
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1259,7 +1175,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
 ; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
-; GFX90A-NEXT:  .LBB43_2:
+; GFX90A-NEXT:  .LBB40_2:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent:
@@ -1269,7 +1185,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB43_2
+; GFX940-NEXT:    s_cbranch_execz .LBB40_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1281,7 +1197,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
 ; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    buffer_inv sc1
-; GFX940-NEXT:  .LBB43_2:
+; GFX940-NEXT:  .LBB40_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
@@ -1296,7 +1212,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB44_3
+; GFX90A-NEXT:    s_cbranch_execz .LBB41_3
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
@@ -1308,7 +1224,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB44_2: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB41_2: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX90A-NEXT:    buffer_wbl2
@@ -1320,8 +1236,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB44_2
-; GFX90A-NEXT:  .LBB44_3:
+; GFX90A-NEXT:    s_cbranch_execnz .LBB41_2
+; GFX90A-NEXT:  .LBB41_3:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system:
@@ -1331,7 +1247,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB44_2
+; GFX940-NEXT:    s_cbranch_execz .LBB41_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1343,7 +1259,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    buffer_inv sc0 sc1
-; GFX940-NEXT:  .LBB44_2:
+; GFX940-NEXT:  .LBB41_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst
@@ -1358,7 +1274,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB45_2
+; GFX90A-NEXT:    s_cbranch_execz .LBB42_2
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1369,7 +1285,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
 ; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
-; GFX90A-NEXT:  .LBB45_2:
+; GFX90A-NEXT:  .LBB42_2:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush:
@@ -1379,7 +1295,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB45_2
+; GFX940-NEXT:    s_cbranch_execz .LBB42_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1391,7 +1307,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
 ; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    buffer_inv sc1
-; GFX940-NEXT:  .LBB45_2:
+; GFX940-NEXT:  .LBB42_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
@@ -1423,7 +1339,7 @@ define double @global_atomic_fadd_f64_rtn_pat(ptr addrspace(1) %ptr, double %dat
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
@@ -1436,7 +1352,7 @@ define double @global_atomic_fadd_f64_rtn_pat(ptr addrspace(1) %ptr, double %dat
 ; GFX90A-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
 ; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB44_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
@@ -1488,7 +1404,7 @@ define double @global_atomic_fadd_f64_rtn_pat_system(ptr addrspace(1) %ptr, doub
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:  .LBB49_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB46_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
@@ -1501,7 +1417,7 @@ define double @global_atomic_fadd_f64_rtn_pat_system(ptr addrspace(1) %ptr, doub
 ; GFX90A-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
 ; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB49_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB46_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
@@ -1568,7 +1484,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB52_3
+; GFX90A-NEXT:    s_cbranch_execz .LBB49_3
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
@@ -1580,7 +1496,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB52_2: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB49_2: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX90A-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
@@ -1590,8 +1506,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB52_2
-; GFX90A-NEXT:  .LBB52_3:
+; GFX90A-NEXT:    s_cbranch_execnz .LBB49_2
+; GFX90A-NEXT:  .LBB49_3:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
@@ -1601,7 +1517,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB52_2
+; GFX940-NEXT:    s_cbranch_execz .LBB49_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
@@ -1613,7 +1529,7 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    buffer_inv sc1
-; GFX940-NEXT:  .LBB52_2:
+; GFX940-NEXT:  .LBB49_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
@@ -1629,7 +1545,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB53_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB50_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], 4.0
@@ -1642,7 +1558,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB53_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB50_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
 ;
@@ -1700,7 +1616,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB55_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB52_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], 4.0
@@ -1714,7 +1630,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB55_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB52_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
 ;
@@ -1740,7 +1656,7 @@ define double @flat_atomic_fadd_f64_rtn_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:  .LBB56_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB53_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
@@ -1753,7 +1669,7 @@ define double @flat_atomic_fadd_f64_rtn_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
 ; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB56_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB53_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
@@ -1805,7 +1721,7 @@ define double @flat_atomic_fadd_f64_rtn_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:  .LBB58_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB55_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
@@ -1819,7 +1735,7 @@ define double @flat_atomic_fadd_f64_rtn_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
 ; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB58_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB55_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
@@ -1896,7 +1812,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX90A-NEXT:  .LBB58_1: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], 4.0
@@ -1907,7 +1823,7 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
 ; GFX90A-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB58_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
 ;
@@ -2123,7 +2039,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB70_2
+; GFX90A-NEXT:    s_cbranch_execz .LBB67_2
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2133,7 +2049,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB70_2:
+; GFX90A-NEXT:  .LBB67_2:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat:
@@ -2143,7 +2059,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB70_2
+; GFX940-NEXT:    s_cbranch_execz .LBB67_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2153,7 +2069,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX940-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB70_2:
+; GFX940-NEXT:  .LBB67_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
@@ -2168,7 +2084,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB71_2
+; GFX90A-NEXT:    s_cbranch_execz .LBB68_2
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2178,7 +2094,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB71_2:
+; GFX90A-NEXT:  .LBB68_2:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush:
@@ -2188,7 +2104,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB71_2
+; GFX940-NEXT:    s_cbranch_execz .LBB68_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2198,7 +2114,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX940-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB71_2:
+; GFX940-NEXT:  .LBB68_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
@@ -2213,7 +2129,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB72_2
+; GFX90A-NEXT:    s_cbranch_execz .LBB69_2
 ; GFX90A-NEXT:  ; %bb.1:
 ; GFX90A-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX90A-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2223,7 +2139,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB72_2:
+; GFX90A-NEXT:  .LBB69_2:
 ; GFX90A-NEXT:    s_endpgm
 ;
 ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
@@ -2233,7 +2149,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB72_2
+; GFX940-NEXT:    s_cbranch_execz .LBB69_2
 ; GFX940-NEXT:  ; %bb.1:
 ; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX940-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
@@ -2243,7 +2159,7 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX940-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB72_2:
+; GFX940-NEXT:  .LBB69_2:
 ; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index 687188ed5ca391d1506e410c36aa74a21bd9c1f1..d4ff845e1edf3a0c9e88de8d761c67611c5aab8f 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -331,15 +331,14 @@ bb3:                                              ; preds = %bb3, %bb
 define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
 ; GFX9-LABEL: sdiv32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
-; GFX9-NEXT:    s_mov_b32 s4, 0
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s3, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
-; GFX9-NEXT:    s_add_i32 s3, s3, s2
-; GFX9-NEXT:    s_xor_b32 s3, s3, s2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT:    s_sub_i32 s5, 0, s3
+; GFX9-NEXT:    s_abs_i32 s2, s4
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_sub_i32 s5, 0, s2
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 31
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
@@ -350,25 +349,25 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:  .LBB2_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s5
-; GFX9-NEXT:    s_mul_i32 s7, s6, s3
-; GFX9-NEXT:    s_sub_i32 s7, s4, s7
+; GFX9-NEXT:    s_mul_hi_u32 s6, s3, s5
+; GFX9-NEXT:    s_mul_i32 s7, s6, s2
+; GFX9-NEXT:    s_sub_i32 s7, s3, s7
 ; GFX9-NEXT:    s_add_i32 s8, s6, 1
-; GFX9-NEXT:    s_sub_i32 s9, s7, s3
-; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX9-NEXT:    s_sub_i32 s9, s7, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX9-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX9-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX9-NEXT:    s_add_i32 s8, s6, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX9-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX9-NEXT:    s_xor_b32 s6, s6, s2
-; GFX9-NEXT:    s_sub_i32 s6, s6, s2
-; GFX9-NEXT:    s_add_i32 s4, s4, 1
+; GFX9-NEXT:    s_xor_b32 s6, s6, s4
+; GFX9-NEXT:    s_sub_i32 s6, s6, s4
+; GFX9-NEXT:    s_add_i32 s3, s3, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, 4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
+; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB2_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
 ; GFX9-NEXT:    s_endpgm
@@ -377,12 +376,11 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x2c
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_ashr_i32 s2, s3, 31
+; GFX10-NEXT:    s_abs_i32 s2, s3
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    s_add_i32 s3, s3, s2
-; GFX10-NEXT:    s_xor_b32 s3, s3, s2
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX10-NEXT:    s_sub_i32 s4, 0, s3
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX10-NEXT:    s_sub_i32 s4, 0, s2
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 31
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
@@ -395,19 +393,19 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX10-NEXT:  .LBB2_1: ; %bb3
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    s_mul_hi_u32 s6, s4, s5
-; GFX10-NEXT:    s_mul_i32 s7, s6, s3
+; GFX10-NEXT:    s_mul_i32 s7, s6, s2
 ; GFX10-NEXT:    s_add_i32 s8, s6, 1
 ; GFX10-NEXT:    s_sub_i32 s7, s4, s7
-; GFX10-NEXT:    s_sub_i32 s9, s7, s3
-; GFX10-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX10-NEXT:    s_sub_i32 s9, s7, s2
+; GFX10-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX10-NEXT:    s_add_i32 s8, s6, 1
-; GFX10-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX10-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX10-NEXT:    s_add_i32 s4, s4, 1
-; GFX10-NEXT:    s_xor_b32 s6, s6, s2
-; GFX10-NEXT:    s_sub_i32 s6, s6, s2
+; GFX10-NEXT:    s_xor_b32 s6, s6, s3
+; GFX10-NEXT:    s_sub_i32 s6, s6, s3
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
@@ -425,22 +423,20 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x2c
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_ashr_i32 s2, s3, 31
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_add_i32 s3, s3, s2
-; GFX11-NEXT:    s_xor_b32 s3, s3, s2
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX11-NEXT:    s_sub_i32 s4, 0, s3
+; GFX11-NEXT:    s_abs_i32 s2, s3
+; GFX11-NEXT:    s_ashr_i32 s3, s3, 31
+; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX11-NEXT:    s_sub_i32 s4, 0, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_mul_i32 s4, s4, s5
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_mul_hi_u32 s6, s5, s4
 ; GFX11-NEXT:    s_mov_b32 s4, 0
 ; GFX11-NEXT:    s_add_i32 s5, s5, s6
@@ -449,21 +445,21 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_mul_hi_u32 s6, s4, s5
-; GFX11-NEXT:    s_mul_i32 s7, s6, s3
+; GFX11-NEXT:    s_mul_i32 s7, s6, s2
 ; GFX11-NEXT:    s_add_i32 s8, s6, 1
 ; GFX11-NEXT:    s_sub_i32 s7, s4, s7
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s9, s7, s3
-; GFX11-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX11-NEXT:    s_sub_i32 s9, s7, s2
+; GFX11-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX11-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX11-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX11-NEXT:    s_add_i32 s8, s6, 1
-; GFX11-NEXT:    s_cmp_ge_u32 s7, s3
+; GFX11-NEXT:    s_cmp_ge_u32 s7, s2
 ; GFX11-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX11-NEXT:    s_add_i32 s4, s4, 1
-; GFX11-NEXT:    s_xor_b32 s6, s6, s2
+; GFX11-NEXT:    s_xor_b32 s6, s6, s3
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s6, s6, s2
+; GFX11-NEXT:    s_sub_i32 s6, s6, s3
 ; GFX11-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, 4
@@ -495,14 +491,12 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX9-LABEL: srem32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s3, 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    s_abs_i32 s2, s2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_sub_i32 s4, 0, s2
-; GFX9-NEXT:    s_mov_b32 s3, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
@@ -524,7 +518,6 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
 ; GFX9-NEXT:    s_add_i32 s3, s3, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, 4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, 0
@@ -537,10 +530,8 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX10-NEXT:    s_abs_i32 s2, s2
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_xor_b32 s2, s2, s3
 ; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX10-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
@@ -581,10 +572,7 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_add_i32 s2, s2, s3
-; GFX11-NEXT:    s_xor_b32 s2, s2, s3
+; GFX11-NEXT:    s_abs_i32 s2, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX11-NEXT:    s_sub_i32 s3, 0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
index f950717c591a96811b65c7a2576db6636729a406..3e6de324924579199b9bd19f06dbf2d24ddf2633 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
@@ -17,15 +17,16 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GCN-NEXT:    s_cbranch_execz .LBB0_14
 ; GCN-NEXT:  ; %bb.1: ; %itofp-if-end
-; GCN-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
-; GCN-NEXT:    v_xor_b32_e32 v0, v5, v0
-; GCN-NEXT:    v_xor_b32_e32 v1, v5, v1
-; GCN-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v5
-; GCN-NEXT:    v_xor_b32_e32 v2, v5, v2
-; GCN-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-NEXT:    v_xor_b32_e32 v6, v5, v3
-; GCN-NEXT:    v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
-; GCN-NEXT:    v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_sub_co_u32_e32 v4, vcc, 0, v0
+; GCN-NEXT:    v_subb_co_u32_e32 v5, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subb_co_u32_e32 v6, vcc, 0, v2, vcc
+; GCN-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; GCN-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
 ; GCN-NEXT:    v_ffbh_u32_e32 v2, v4
 ; GCN-NEXT:    v_add_u32_e32 v2, 32, v2
 ; GCN-NEXT:    v_ffbh_u32_e32 v6, v5
@@ -40,7 +41,6 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    v_sub_u32_e32 v6, 0x80, v7
 ; GCN-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
 ; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
-; GCN-NEXT:    ; implicit-def: $vgpr8
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:  ; %bb.2: ; %itofp-if-else
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index c6aa2182aec80c71f86ac29aefaa0adfec199349..c5198cdb421a502be9689f9ba7761c9400f232d5 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -13,15 +13,16 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_14
 ; SDAG-NEXT:  ; %bb.1: ; %itofp-if-end
-; SDAG-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
-; SDAG-NEXT:    v_xor_b32_e32 v0, v5, v0
-; SDAG-NEXT:    v_xor_b32_e32 v1, v5, v1
-; SDAG-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v5
-; SDAG-NEXT:    v_xor_b32_e32 v2, v5, v2
-; SDAG-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v6, v5, v3
-; SDAG-NEXT:    v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
-; SDAG-NEXT:    v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
+; SDAG-NEXT:    v_sub_co_u32_e32 v4, vcc, 0, v0
+; SDAG-NEXT:    v_subb_co_u32_e32 v5, vcc, 0, v1, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v6, vcc, 0, v2, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT:    ; implicit-def: $vgpr8
+; SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
 ; SDAG-NEXT:    v_ffbh_u32_e32 v2, v4
 ; SDAG-NEXT:    v_add_u32_e32 v2, 32, v2
 ; SDAG-NEXT:    v_ffbh_u32_e32 v6, v5
@@ -36,7 +37,6 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    v_sub_u32_e32 v6, 0x80, v7
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
 ; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
-; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SDAG-NEXT:  ; %bb.2: ; %itofp-if-else
@@ -524,16 +524,17 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_14
 ; SDAG-NEXT:  ; %bb.1: ; %itofp-if-end
-; SDAG-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
-; SDAG-NEXT:    v_xor_b32_e32 v4, v0, v4
-; SDAG-NEXT:    v_xor_b32_e32 v5, v0, v5
-; SDAG-NEXT:    v_sub_co_u32_e32 v4, vcc, v4, v0
-; SDAG-NEXT:    v_xor_b32_e32 v2, v0, v2
-; SDAG-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v0, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v1, v0, v3
-; SDAG-NEXT:    v_subb_co_u32_e32 v6, vcc, v2, v0, vcc
-; SDAG-NEXT:    v_subb_co_u32_e32 v7, vcc, v1, v0, vcc
+; SDAG-NEXT:    v_sub_co_u32_e32 v0, vcc, 0, v4
+; SDAG-NEXT:    v_subb_co_u32_e32 v1, vcc, 0, v5, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v6, vcc, 0, v2, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT:    ; implicit-def: $vgpr10
+; SDAG-NEXT:    v_cndmask_b32_e32 v6, v2, v6, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v7, v3, v7, vcc
 ; SDAG-NEXT:    v_ffbh_u32_e32 v0, v6
+; SDAG-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc
 ; SDAG-NEXT:    v_add_u32_e32 v0, 32, v0
 ; SDAG-NEXT:    v_ffbh_u32_e32 v1, v7
 ; SDAG-NEXT:    v_min_u32_e32 v0, v0, v1
@@ -547,7 +548,6 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    v_sub_u32_e32 v8, 0x80, v9
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v9
 ; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 54, v8
-; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -1103,15 +1103,16 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB4_14
 ; SDAG-NEXT:  ; %bb.1: ; %itofp-if-end
-; SDAG-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
-; SDAG-NEXT:    v_xor_b32_e32 v0, v5, v0
-; SDAG-NEXT:    v_xor_b32_e32 v1, v5, v1
-; SDAG-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v5
-; SDAG-NEXT:    v_xor_b32_e32 v2, v5, v2
-; SDAG-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; SDAG-NEXT:    v_xor_b32_e32 v6, v5, v3
-; SDAG-NEXT:    v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
-; SDAG-NEXT:    v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
+; SDAG-NEXT:    v_sub_co_u32_e32 v4, vcc, 0, v0
+; SDAG-NEXT:    v_subb_co_u32_e32 v5, vcc, 0, v1, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v6, vcc, 0, v2, vcc
+; SDAG-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; SDAG-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT:    ; implicit-def: $vgpr8
+; SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
 ; SDAG-NEXT:    v_ffbh_u32_e32 v2, v4
 ; SDAG-NEXT:    v_add_u32_e32 v2, 32, v2
 ; SDAG-NEXT:    v_ffbh_u32_e32 v6, v5
@@ -1126,7 +1127,6 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    v_sub_u32_e32 v6, 0x80, v7
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
 ; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
-; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SDAG-NEXT:  ; %bb.2: ; %itofp-if-else
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 0db88d1c095d30aa171de5afdafb8721779fcaed..08cf83fd2bd0fbf0dfcaf29a540ee9eca50c8474 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -31,6 +31,7 @@
 ; GCN-O0-NEXT:    AMDGPU Remove Incompatible Functions
 ; GCN-O0-NEXT:    AMDGPU Printf lowering
 ; GCN-O0-NEXT:    Lower ctors and dtors for AMDGPU
+; GCN-O0-NEXT:    Expand variadic functions
 ; GCN-O0-NEXT:    AMDGPU Inline All Functions
 ; GCN-O0-NEXT:    Inliner for always_inline functions
 ; GCN-O0-NEXT:      FunctionPass Manager
@@ -178,6 +179,7 @@
 ; GCN-O1-NEXT:    AMDGPU Remove Incompatible Functions
 ; GCN-O1-NEXT:    AMDGPU Printf lowering
 ; GCN-O1-NEXT:    Lower ctors and dtors for AMDGPU
+; GCN-O1-NEXT:    Expand variadic functions
 ; GCN-O1-NEXT:    AMDGPU Inline All Functions
 ; GCN-O1-NEXT:    Inliner for always_inline functions
 ; GCN-O1-NEXT:      FunctionPass Manager
@@ -454,6 +456,7 @@
 ; GCN-O1-OPTS-NEXT:    AMDGPU Remove Incompatible Functions
 ; GCN-O1-OPTS-NEXT:    AMDGPU Printf lowering
 ; GCN-O1-OPTS-NEXT:    Lower ctors and dtors for AMDGPU
+; GCN-O1-OPTS-NEXT:    Expand variadic functions
 ; GCN-O1-OPTS-NEXT:    AMDGPU Inline All Functions
 ; GCN-O1-OPTS-NEXT:    Inliner for always_inline functions
 ; GCN-O1-OPTS-NEXT:      FunctionPass Manager
@@ -760,6 +763,7 @@
 ; GCN-O2-NEXT:    Lower ctors and dtors for AMDGPU
 ; GCN-O2-NEXT:    FunctionPass Manager
 ; GCN-O2-NEXT:      AMDGPU Image Intrinsic Optimizer
+; GCN-O2-NEXT:    Expand variadic functions
 ; GCN-O2-NEXT:    AMDGPU Inline All Functions
 ; GCN-O2-NEXT:    Inliner for always_inline functions
 ; GCN-O2-NEXT:      FunctionPass Manager
@@ -1070,6 +1074,7 @@
 ; GCN-O3-NEXT:    Lower ctors and dtors for AMDGPU
 ; GCN-O3-NEXT:    FunctionPass Manager
 ; GCN-O3-NEXT:      AMDGPU Image Intrinsic Optimizer
+; GCN-O3-NEXT:    Expand variadic functions
 ; GCN-O3-NEXT:    AMDGPU Inline All Functions
 ; GCN-O3-NEXT:    Inliner for always_inline functions
 ; GCN-O3-NEXT:      FunctionPass Manager
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll
deleted file mode 100644
index 2b0584d39a3be46f0f083bcf361852c360ad51d5..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll
+++ /dev/null
@@ -1,82 +0,0 @@
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,PREGFX12
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1031 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,PREGFX12
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GFX12PLUS
-
-declare i32 @llvm.amdgcn.buffer.atomic.csub(i32, <4 x i32>, i32, i32, i1)
-declare i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1), i32)
-
-; GCN-LABEL: {{^}}buffer_atomic_csub_rtn:
-; PREGFX12: buffer_atomic_csub v0, v1, s[0:3], 0 idxen glc
-; GFX12PLUS: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null idxen th:TH_ATOMIC_RETURN
-define amdgpu_ps void @buffer_atomic_csub_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
-main_body:
-  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_csub_no_rtn:
-; PREGFX12: buffer_atomic_csub v0, v1, s[0:3], 0 idxen
-; GFX12PLUS: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null idxen
-define amdgpu_ps void @buffer_atomic_csub_no_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) #0 {
-main_body:
-  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_csub_off4_slc_rtn:
-; PREGFX12: buffer_atomic_csub v0, v1, s[0:3], 0 idxen offset:4 glc slc
-; GFX12PLUS: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
-define amdgpu_ps void @buffer_atomic_csub_off4_slc_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
-main_body:
-  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_csub_off4_slc_no_rtn:
-; PREGFX12: buffer_atomic_csub v0, v1, s[0:3], 0 idxen offset:4 slc
-; GFX12PLUS: buffer_atomic_sub_clamp_u32 v0, v1, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT
-define amdgpu_ps void @buffer_atomic_csub_off4_slc_no_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) #0 {
-main_body:
-  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_csub_rtn:
-; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9:]+}}, s{{\[[0-9]+:[0-9]+\]}} glc
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
-define amdgpu_kernel void @global_atomic_csub_rtn(ptr addrspace(1) %ptr, i32 %data) {
-main_body:
-  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_csub_no_rtn:
-; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v1, s[0:1]
-define amdgpu_kernel void @global_atomic_csub_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
-main_body:
-  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_csub_off4_rtn:
-; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4 glc
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v0, v1, s[0:1] offset:4 th:TH_ATOMIC_RETURN
-define amdgpu_kernel void @global_atomic_csub_off4_rtn(ptr addrspace(1) %ptr, i32 %data) {
-main_body:
-  %p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
-  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %p, i32 %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_csub_off4_no_rtn:
-; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v1, s[0:1] offset:4
-define amdgpu_kernel void @global_atomic_csub_off4_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
-main_body:
-  %p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
-  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %p, i32 %data)
-  ret void
-}
-
-attributes #0 = { "target-features"="+atomic-csub-no-rtn-insts" }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.gfx90a.ll
deleted file mode 100644
index 1e1bc2bbbd26aa0fbab0224a54cc2aabe4e9fbe7..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.gfx90a.ll
+++ /dev/null
@@ -1,93 +0,0 @@
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs | FileCheck %s -check-prefix=GFX90A
-; RUN: not --crash llc < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
-
-declare float @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1)
-declare <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i1)
-declare float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1), float)
-declare <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1), <2 x half>)
-
-; GFX908:  LLVM ERROR: Cannot select: {{.+}}: f32,ch = BUFFER_ATOMIC_FADD
-
-; GFX90A-LABEL: {{^}}buffer_atomic_add_f32:
-; GFX90A: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen glc
-define amdgpu_ps float @buffer_atomic_add_f32(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
-main_body:
-  %ret = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret float %ret
-}
-
-; GFX90A-LABEL: {{^}}buffer_atomic_add_f32_off4_slc:
-; GFX90A: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen offset:4 glc slc
-define amdgpu_ps float @buffer_atomic_add_f32_off4_slc(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
-main_body:
-  %ret = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret float %ret
-}
-
-; GFX90A-LABEL: {{^}}buffer_atomic_pk_add_v2f16:
-; GFX90A: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen glc
-define amdgpu_ps <2 x half> @buffer_atomic_pk_add_v2f16(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret <2 x half> %ret
-}
-
-; GFX90A-LABEL: {{^}}buffer_atomic_pk_add_v2f16_off4_slc:
-; GFX90A: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen offset:4 glc slc
-define amdgpu_ps <2 x half> @buffer_atomic_pk_add_v2f16_off4_slc(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret <2 x half> %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_add_f32:
-; GFX90A: global_atomic_add_f32 v0, v[0:1], v2, off glc
-define amdgpu_ps float @global_atomic_add_f32(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %ptr, float %data)
-  ret float %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_add_f32_off4:
-; GFX90A: global_atomic_add_f32 v0, v[0:1], v2, off offset:4 glc
-define amdgpu_ps float @global_atomic_add_f32_off4(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %p = getelementptr float, ptr addrspace(1) %ptr, i64 1
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %p, float %data)
-  ret float %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_add_f32_offneg4:
-; GFX90A: global_atomic_add_f32 v0, v[0:1], v2, off offset:-4 glc
-define amdgpu_ps float @global_atomic_add_f32_offneg4(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %p = getelementptr float, ptr addrspace(1) %ptr, i64 -1
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %p, float %data)
-  ret float %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_pk_add_v2f16:
-; GFX90A: global_atomic_pk_add_f16 v0, v[0:1], v2, off glc
-define amdgpu_ps <2 x half> @global_atomic_pk_add_v2f16(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %ptr, <2 x half> %data)
-  ret <2 x half> %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_pk_add_v2f16_off4:
-; GFX90A: global_atomic_pk_add_f16 v0, v[0:1], v2, off offset:4 glc
-define amdgpu_ps <2 x half> @global_atomic_pk_add_v2f16_off4(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %p = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 1
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %p, <2 x half> %data)
-  ret <2 x half> %ret
-}
-
-; GFX90A-LABEL: {{^}}global_atomic_pk_add_v2f16_offneg4:
-; GFX90A: global_atomic_pk_add_f16 v0, v[0:1], v2, off offset:-4 glc
-define amdgpu_ps <2 x half> @global_atomic_pk_add_v2f16_offneg4(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %p = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -1
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %p, <2 x half> %data)
-  ret <2 x half> %ret
-}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.ll
deleted file mode 100644
index bd07dd137ac492fb4b0332b7b5f2bd7f1839446c..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.ll
+++ /dev/null
@@ -1,111 +0,0 @@
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs -amdgpu-atomic-optimizer-strategy=DPP | FileCheck %s -check-prefix=GCN
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -amdgpu-atomic-optimizer-strategy=DPP | FileCheck %s -check-prefix=GCN
-
-declare float @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1)
-declare <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i1)
-declare float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1), float)
-declare <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1), <2 x half>)
-declare float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr, float)
-
-; GCN-LABEL: {{^}}buffer_atomic_add_f32:
-; GCN: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_atomic_add_f32(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
-main_body:
-  %ret = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_add_f32_off4_slc:
-; GCN: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen offset:4 slc
-define amdgpu_ps void @buffer_atomic_add_f32_off4_slc(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
-main_body:
-  %ret = call float @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_pk_add_v2f16:
-; GCN: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_atomic_pk_add_v2f16(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_atomic_pk_add_v2f16_off4_slc:
-; GCN: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen offset:4 slc
-define amdgpu_ps void @buffer_atomic_pk_add_v2f16_off4_slc(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_add_f32:
-; GCN: global_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
-define amdgpu_kernel void @global_atomic_add_f32(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %ptr, float %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_add_f32_off4:
-; GCN: global_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
-define amdgpu_kernel void @global_atomic_add_f32_off4(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %p = getelementptr float, ptr addrspace(1) %ptr, i64 1
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %p, float %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_add_f32_offneg4:
-; GCN: global_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:-4
-define amdgpu_kernel void @global_atomic_add_f32_offneg4(ptr addrspace(1) %ptr, float %data) {
-main_body:
-  %p = getelementptr float, ptr addrspace(1) %ptr, i64 -1
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %p, float %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_pk_add_v2f16:
-; GCN: global_atomic_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]$}}
-define amdgpu_kernel void @global_atomic_pk_add_v2f16(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %ptr, <2 x half> %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_pk_add_v2f16_off4:
-; GCN: global_atomic_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
-define amdgpu_kernel void @global_atomic_pk_add_v2f16_off4(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %p = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 1
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %p, <2 x half> %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}global_atomic_pk_add_v2f16_offneg4:
-; GCN: global_atomic_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:-4{{$}}
-define amdgpu_kernel void @global_atomic_pk_add_v2f16_offneg4(ptr addrspace(1) %ptr, <2 x half> %data) {
-main_body:
-  %p = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -1
-  %ret = call <2 x half> @llvm.amdgcn.global.atomic.fadd.v2f16.p1.v2f16(ptr addrspace(1) %p, <2 x half> %data)
-  ret void
-}
-
-; Make sure this artificially selects with an incorrect subtarget, but
-; the feature set.
-; GCN-LABEL: {{^}}global_atomic_fadd_f32_wrong_subtarget:
-; GCN: global_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]$}}
-define amdgpu_kernel void @global_atomic_fadd_f32_wrong_subtarget(ptr addrspace(1) %ptr, float %data) #0 {
-  %ret = call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %ptr, float %data)
-  ret void
-}
-
-; GCN-LABEL: {{^}}flat_atomic_fadd_f32_wrong_subtarget:
-; GCN: flat_atomic_add_f32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
-define amdgpu_kernel void @flat_atomic_fadd_f32_wrong_subtarget(ptr %ptr, float %data) #1 {
-  %ret = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %ptr, float %data)
-  ret void
-}
-
-attributes #0 = { "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts"}
-attributes #1 = { "target-cpu"="gfx803" "target-features"="+flat-atomic-fadd-f32-inst"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll
deleted file mode 100644
index eed648f167f3980c9f18f38f93bd884534c52521..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll
+++ /dev/null
@@ -1,209 +0,0 @@
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -amdgpu-atomic-optimizer-strategy=None -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=SICI
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=VI
-
-;CHECK-LABEL: {{^}}test1:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_atomic_swap v0, off, s[0:3], 0 glc
-;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1ffc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap v0, v2, s[0:3], 0 offen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap v0, v[1:2], s[0:3], 0 idxen offen glc
-;SICI: v_mov_b32_e32 v1, 0x2000
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap v0, v2, s[0:3], 0 offen offset:42 glc
-;CHECK-DAG: s_waitcnt vmcnt(0)
-;SICI: buffer_atomic_swap v0, v1, s[0:3], 0 offen glc
-;VI: buffer_atomic_swap v0, off, s[0:3], [[SOFS]] offset:4 glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap v0, off, s[0:3], 0{{$}}
-define amdgpu_ps float @test1(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex, i32 %voffset) {
-main_body:
-  %o1 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %data, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %o2 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o1, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %o3 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o2, <4 x i32> %rsrc, i32 0, i32 %voffset, i1 0)
-  %o4 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o3, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i1 0)
-  %ofs.5 = add i32 %voffset, 42
-  %o5 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o4, <4 x i32> %rsrc, i32 0, i32 %ofs.5, i1 0)
-  %o6 = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o5, <4 x i32> %rsrc, i32 0, i32 8192, i1 0)
-  %unused = call i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32 %o6, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %out = bitcast i32 %o6 to float
-  ret float %out
-}
-
-;CHECK-LABEL: {{^}}test11:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_atomic_swap_x2 v[3:4], off, s[0:3], 0 glc
-;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1ffc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap_x2 v[3:4], v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap_x2 v[3:4], v2, s[0:3], 0 offen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap_x2 v[3:4], v[1:2], s[0:3], 0 idxen offen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap_x2 v[3:4], v2, s[0:3], 0 offen offset:42 glc
-;CHECK-DAG: s_waitcnt vmcnt(0)
-;SICI: buffer_atomic_swap_x2 v[3:4], v0, s[0:3], 0 offen glc
-;VI: buffer_atomic_swap_x2 v[3:4], off, s[0:3], [[SOFS]] offset:4 glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_swap_x2 v[3:4], off, s[0:3], 0{{$}}
-define amdgpu_ps float @test11(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex, i32 %voffset) {
-main_body:
-  %o0 = sext i32 %data to i64
-  %o1 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o0, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %o2 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o1, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %o3 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o2, <4 x i32> %rsrc, i32 0, i32 %voffset, i1 0)
-  %o4 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o3, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i1 0)
-  %ofs.5 = add i32 %voffset, 42
-  %o5 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o4, <4 x i32> %rsrc, i32 0, i32 %ofs.5, i1 0)
-  %o6 = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o5, <4 x i32> %rsrc, i32 0, i32 8192, i1 0)
-  %unused = call i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64 %o6, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %o7 = trunc i64 %o6 to i32
-  %out = bitcast i32 %o7 to float
-  ret float %out
-}
-
-;CHECK-LABEL: {{^}}test2:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_atomic_add v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_sub v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_smin v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_umin v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_smax v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_umax v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_and v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_or v0, v1, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_xor v0, v1, s[0:3], 0 idxen glc
-define amdgpu_ps float @test2(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
-main_body:
-  %t1 = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t2 = call i32 @llvm.amdgcn.buffer.atomic.sub.i32(i32 %t1, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t3 = call i32 @llvm.amdgcn.buffer.atomic.smin.i32(i32 %t2, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t4 = call i32 @llvm.amdgcn.buffer.atomic.umin.i32(i32 %t3, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t5 = call i32 @llvm.amdgcn.buffer.atomic.smax.i32(i32 %t4, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t6 = call i32 @llvm.amdgcn.buffer.atomic.umax.i32(i32 %t5, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t7 = call i32 @llvm.amdgcn.buffer.atomic.and.i32(i32 %t6, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t8 = call i32 @llvm.amdgcn.buffer.atomic.or.i32(i32 %t7, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t9 = call i32 @llvm.amdgcn.buffer.atomic.xor.i32(i32 %t8, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %out = bitcast i32 %t9 to float
-  ret float %out
-}
-
-;CHECK-LABEL: {{^}}test3:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_smin_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_umin_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_smax_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_umax_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_and_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_or_x2 v[0:1], v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen glc
-define amdgpu_ps float @test3(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
-main_body:
-  %t0 = sext i32 %data to i64
-  %t1 = call i64 @llvm.amdgcn.buffer.atomic.add.i64(i64 %t0, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t2 = call i64 @llvm.amdgcn.buffer.atomic.sub.i64(i64 %t1, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t3 = call i64 @llvm.amdgcn.buffer.atomic.smin.i64(i64 %t2, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t4 = call i64 @llvm.amdgcn.buffer.atomic.umin.i64(i64 %t3, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t5 = call i64 @llvm.amdgcn.buffer.atomic.smax.i64(i64 %t4, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t6 = call i64 @llvm.amdgcn.buffer.atomic.umax.i64(i64 %t5, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t7 = call i64 @llvm.amdgcn.buffer.atomic.and.i64(i64 %t6, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t8 = call i64 @llvm.amdgcn.buffer.atomic.or.i64(i64 %t7, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t9 = call i64 @llvm.amdgcn.buffer.atomic.xor.i64(i64 %t8, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %t10 = trunc i64 %t9 to i32
-  %out = bitcast i32 %t10 to float
-  ret float %out
-}
-
-; Ideally, we would teach tablegen & friends that cmpswap only modifies the
-; first vgpr. Since we don't do that yet, the register allocator will have to
-; create copies which we don't bother to track here.
-;
-;CHECK-LABEL: {{^}}test4:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 0 glc
-;CHECK: s_waitcnt vmcnt(0)
-;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1ffc
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v2, s[0:3], 0 idxen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v3, s[0:3], 0 offen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v[2:3], s[0:3], 0 idxen offen glc
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v3, s[0:3], 0 offen offset:44 glc
-;CHECK-DAG: s_waitcnt vmcnt(0)
-;SICI: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen glc
-;VI: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[SOFS]] offset:4 glc
-define amdgpu_ps float @test4(<4 x i32> inreg %rsrc, i32 %data, i32 %cmp, i32 %vindex, i32 %voffset) {
-main_body:
-  %o1 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %data, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %o2 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o1, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
-  %o3 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o2, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 %voffset, i1 0)
-  %o4 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o3, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i1 0)
-  %ofs.5 = add i32 %voffset, 44
-  %o5 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o4, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 %ofs.5, i1 0)
-  %o6 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o5, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 8192, i1 0)
-
-; Detecting the no-return variant doesn't work right now because of how the
-; intrinsic is replaced by an instruction that feeds into an EXTRACT_SUBREG.
-; Since there probably isn't a reasonable use-case of cmpswap that discards
-; the return value, that seems okay.
-;
-;  %unused = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o6, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
-  %out = bitcast i32 %o6 to float
-  ret float %out
-}
-
-;CHECK-LABEL: {{^}}test7:
-;CHECK: buffer_atomic_add v0,
-define amdgpu_ps float @test7() {
-main_body:
-  %v = call i32 @llvm.amdgcn.buffer.atomic.add.i32(i32 1, <4 x i32> undef, i32 0, i32 4, i1 false)
-  %v.float = bitcast i32 %v to float
-  ret float %v.float
-}
-
-declare i32 @llvm.amdgcn.buffer.atomic.swap.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.sub.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.smin.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.umin.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.smax.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.umax.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.and.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.or.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.xor.i32(i32, <4 x i32>, i32, i32, i1) #0
-declare i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32, i32, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.swap.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.add.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.sub.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.smin.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.umin.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.smax.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.umax.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.and.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.or.i64(i64, <4 x i32>, i32, i32, i1) #0
-declare i64 @llvm.amdgcn.buffer.atomic.xor.i64(i64, <4 x i32>, i32, i32, i1) #0
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.dwordx3.ll
index fdbe6db8da14e5333c2de3dd263f37ed69951c30..659842afad485ffe3fdc9f7e966c9fb3cb64826d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.dwordx3.ll
@@ -1,26 +1,6 @@
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx600 -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,SI
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,GCNX3
 
-;CHECK-LABEL: {{^}}buffer_load_format_immoffs_x3:
-;SI: buffer_load_format_xyzw v[0:3], off, s[0:3], 0 offset:42
-;GCNX3: buffer_load_format_xyz v[0:2], off, s[0:3], 0 offset:42
-;CHECK: s_waitcnt
-define amdgpu_ps <3 x float> @buffer_load_format_immoffs_x3(<4 x i32> inreg) {
-main_body:
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret <3 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs_x3:
-;SI: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:40
-;GCNX3: buffer_load_dwordx3 v[0:2], off, s[0:3], 0 offset:40
-;CHECK: s_waitcnt
-define amdgpu_ps <3 x float> @buffer_load_immoffs_x3(<4 x i32> inreg) {
-main_body:
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %0, i32 0, i32 40, i1 0, i1 0)
-  ret <3 x float> %data
-}
-
 ;CHECK-LABEL: {{^}}buffer_raw_load_immoffs_x3:
 ;SI: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:40
 ;GCNX3: buffer_load_dwordx3 v[0:2], off, s[0:3], 0 offset:40
@@ -81,8 +61,6 @@ main_body:
   ret <3 x float> %data
 }
 
-declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #0
-declare <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32>, i32, i32, i1, i1) #0
 declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32) #0
 declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32) #0
 declare <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.d16.ll
deleted file mode 100644
index 3d67dfdd674d4787706ed374d70d1c3c5b37374c..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.d16.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx810 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
-
-; GCN-LABEL: {{^}}buffer_load_format_d16_x:
-; GCN: buffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0
-define amdgpu_ps half @buffer_load_format_d16_x(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call half @llvm.amdgcn.buffer.load.format.f16(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  ret half %data
-}
-
-; GCN-LABEL: {{^}}buffer_load_format_d16_xy:
-; UNPACKED: buffer_load_format_d16_xy v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: buffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0
-; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[FULL]]
-define amdgpu_ps half @buffer_load_format_d16_xy(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <2 x half> @llvm.amdgcn.buffer.load.format.v2f16(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  %elt = extractelement <2 x half> %data, i32 1
-  ret half %elt
-}
-
-; GCN-LABEL: {{^}}buffer_load_format_d16_xyz:
-; UNPACKED: buffer_load_format_d16_xyz v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: buffer_load_format_d16_xyz v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0
-; PACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-define amdgpu_ps half @buffer_load_format_d16_xyz(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <3 x half> @llvm.amdgcn.buffer.load.format.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  %elt = extractelement <3 x half> %data, i32 2
-  ret half %elt
-}
-
-; GCN-LABEL: {{^}}buffer_load_format_d16_xyzw:
-; UNPACKED: buffer_load_format_d16_xyzw v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: buffer_load_format_d16_xyzw v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0
-; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[HI]]
-define amdgpu_ps half @buffer_load_format_d16_xyzw(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <4 x half> @llvm.amdgcn.buffer.load.format.v4f16(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  %elt = extractelement <4 x half> %data, i32 3
-  ret half %elt
-}
-
-declare half @llvm.amdgcn.buffer.load.format.f16(<4 x i32>, i32, i32, i1, i1)
-declare <2 x half> @llvm.amdgcn.buffer.load.format.v2f16(<4 x i32>, i32, i32, i1, i1)
-declare <3 x half> @llvm.amdgcn.buffer.load.format.v3f16(<4 x i32>, i32, i32, i1, i1)
-declare <4 x half> @llvm.amdgcn.buffer.load.format.v4f16(<4 x i32>, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll
deleted file mode 100644
index 6851302fdcda34cf1be9be61e13f8b4fc46ccd94..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll
+++ /dev/null
@@ -1,133 +0,0 @@
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=SICI
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=VI
-
-;CHECK-LABEL: {{^}}buffer_load:
-;CHECK: buffer_load_format_xyzw v[0:3], off, s[0:3], 0
-;CHECK: buffer_load_format_xyzw v[4:7], off, s[0:3], 0 glc
-;CHECK: buffer_load_format_xyzw v[8:11], off, s[0:3], 0 slc
-;CHECK: s_waitcnt
-define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> inreg) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i1 0, i1 0)
-  %data_glc = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i1 1, i1 0)
-  %data_slc = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i1 0, i1 1)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
-  %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
-  %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
-  ret {<4 x float>, <4 x float>, <4 x float>} %r2
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs:
-;CHECK: buffer_load_format_xyzw v[0:3], off, s[0:3], 0 offset:42
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs_large:
-;SICI: v_mov_b32_e32 [[VOFS:v[0-9]+]], 0x1038
-;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[VOFS]], s[0:3], 0 offen
-;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 60 offset:4092
-;VI-DAG: s_movk_i32 [[OFS1:s[0-9]+]], 0x7ffc
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS1]] offset:4092
-;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen
-;VI-DAG: s_mov_b32 [[OFS2:s[0-9]+]], 0x8ffc
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS2]] offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
-main_body:
-  %d.0 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 4152, i1 0, i1 0)
-  %d.1 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 36856, i1 0, i1 0)
-  %d.2 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 36864, i1 0, i1 0)
-  %d.3 = fadd <4 x float> %d.0, %d.1
-  %data = fadd <4 x float> %d.2, %d.3
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs_reuse:
-;VI: s_movk_i32 [[OFS:s[0-9]+]], 0xffc
-;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:68
-;VI-NOT: s_mov
-;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:84
-;VI: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_immoffs_reuse(<4 x i32> inreg) {
-main_body:
-  %d.0 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 4160, i1 0, i1 0)
-  %d.1 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 4176, i1 0, i1 0)
-  %data = fadd <4 x float> %d.0, %d.1
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_idx:
-;CHECK: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %1, i32 0, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ofs:
-;CHECK: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 %1, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ofs_imm:
-;CHECK: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 offen offset:60
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) {
-main_body:
-  %ofs = add i32 %1, 60
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 %ofs, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_both:
-;CHECK: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %1, i32 %2, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_both_reversed:
-;CHECK: v_mov_b32_e32 v2, v0
-;CHECK: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %2, i32 %1, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x:
-;CHECK: buffer_load_format_x v0, off, s[0:3], 0
-;CHECK: s_waitcnt
-define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  ret float %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_xy:
-;CHECK: buffer_load_format_xy v[0:1], off, s[0:3], 0
-;CHECK: s_waitcnt
-define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  ret <2 x float> %data
-}
-
-declare float @llvm.amdgcn.buffer.load.format.f32(<4 x i32>, i32, i32, i1, i1) #0
-declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i1, i1) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #0
-
-attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
deleted file mode 100644
index a209dcfe3a7a0e15830681761ada8fc49a9fa0d2..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
+++ /dev/null
@@ -1,476 +0,0 @@
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=SICI
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=VI
-
-;CHECK-LABEL: {{^}}buffer_load:
-;CHECK: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
-;CHECK: buffer_load_dwordx4 v[4:7], off, s[0:3], 0 glc
-;CHECK: buffer_load_dwordx4 v[8:11], off, s[0:3], 0 slc
-;CHECK: s_waitcnt
-define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> inreg) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i1 0, i1 0)
-  %data_glc = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i1 1, i1 0)
-  %data_slc = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i1 0, i1 1)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
-  %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
-  %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
-  ret {<4 x float>, <4 x float>, <4 x float>} %r2
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs:
-;CHECK: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:40
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 40, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_immoffs_large:
-;SICI: buffer_load_dwordx4 v[0:3], {{v[0-9]+}}, s[0:3], 0 offen
-;VI: s_movk_i32 [[OFFSET:s[0-9]+]], 0x1ffc
-;VI: buffer_load_dwordx4 v[0:3], off, s[0:3], [[OFFSET]] offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 8192, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_idx:
-;CHECK: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 idxen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 %1, i32 0, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ofs:
-;CHECK: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 %1, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ofs_imm:
-;CHECK: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:60
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) {
-main_body:
-  %ofs = add i32 %1, 60
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 %ofs, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_both:
-;CHECK: buffer_load_dwordx4 v[0:3], v[0:1], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 %1, i32 %2, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_both_reversed:
-;CHECK: v_mov_b32_e32 v2, v0
-;CHECK: buffer_load_dwordx4 v[0:3], v[1:2], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i32) {
-main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 %2, i32 %1, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x1:
-;CHECK: buffer_load_dword v0, v[0:1], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps float @buffer_load_x1(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) {
-main_body:
-  %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 0, i1 0)
-  ret float %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x2:
-;CHECK: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 idxen offen
-;CHECK: s_waitcnt
-define amdgpu_ps <2 x float> @buffer_load_x2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) {
-main_body:
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 0, i1 0)
-  ret <2 x float> %data
-}
-
-;CHECK-LABEL: {{^}}buffer_load_negative_offset:
-;CHECK: v_add_{{[iu]}}32_e32 [[VOFS:v[0-9]+]], vcc, -16, v0
-;CHECK: buffer_load_dwordx4 v[0:3], [[VOFS]], s[0:3], 0 offen
-define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) {
-main_body:
-  %ofs.1 = add i32 %ofs, -16
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 %ofs.1, i1 0, i1 0)
-  ret <4 x float> %data
-}
-
-; SI won't merge ds memory operations, because of the signed offset bug, so
-; we only have check lines for VI.
-; CHECK-LABEL: buffer_load_mmo:
-; VI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; VI: ds_write2_b32 v{{[0-9]+}}, [[ZERO]], [[ZERO]] offset1:4
-define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, ptr addrspace(3) %lds) {
-entry:
-  store float 0.0, ptr addrspace(3) %lds
-  %val = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0)
-  %tmp2 = getelementptr float, ptr addrspace(3) %lds, i32 4
-  store float 0.0, ptr addrspace(3) %tmp2
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
-main_body:
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 8
-  %a3 = add i32 %a, 12
-  %a4 = add i32 %a, 16
-  %a5 = add i32 %a, 28
-  %a6 = add i32 %a, 32
-  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a3, i1 0, i1 0)
-  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a4, i1 0, i1 0)
-  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a5, i1 0, i1 0)
-  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a6, i1 0, i1 0)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged_glc_slc:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
-;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:12 glc{{$}}
-;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 glc slc{{$}}
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x1_offen_merged_glc_slc(<4 x i32> inreg %rsrc, i32 %a) {
-main_body:
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 8
-  %a3 = add i32 %a, 12
-  %a4 = add i32 %a, 16
-  %a5 = add i32 %a, 28
-  %a6 = add i32 %a, 32
-  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a3, i1 1, i1 0)
-  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a4, i1 1, i1 0)
-  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a5, i1 1, i1 1)
-  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a6, i1 1, i1 1)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x2_offen_merged:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
-main_body:
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 12
-  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  %vr2 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  %r1 = extractelement <2 x float> %vr1, i32 0
-  %r2 = extractelement <2 x float> %vr1, i32 1
-  %r3 = extractelement <2 x float> %vr2, i32 0
-  %r4 = extractelement <2 x float> %vr2, i32 1
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x3_offen_merged:
-;CHECK-NEXT: %bb.
-;VI-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x3_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
-main_body:
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 12
-  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  %r1 = extractelement <2 x float> %vr1, i32 0
-  %r2 = extractelement <2 x float> %vr1, i32 1
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float undef, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x1_offset_merged:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x1_offset_merged(<4 x i32> inreg %rsrc) {
-main_body:
-  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 28, i1 0, i1 0)
-  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 32, i1 0, i1 0)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x2_offset_merged:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x2_offset_merged(<4 x i32> inreg %rsrc) {
-main_body:
-  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  %vr2 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  %r1 = extractelement <2 x float> %vr1, i32 0
-  %r2 = extractelement <2 x float> %vr1, i32 1
-  %r3 = extractelement <2 x float> %vr2, i32 0
-  %r4 = extractelement <2 x float> %vr2, i32 1
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_x3_offset_merged:
-;CHECK-NEXT: %bb.
-;VI-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x3_offset_merged(<4 x i32> inreg %rsrc) {
-main_body:
-  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  %r1 = extractelement <2 x float> %vr1, i32 0
-  %r2 = extractelement <2 x float> %vr1, i32 1
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float undef, i1 true, i1 true)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ubyte:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ubyte(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %val = uitofp i8 %tmp to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ushort:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, off, s[0:3], 0 offset:16
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_cvt_f32_u32_e32 v0, v0
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ushort(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  %tmp2 = zext i16 %tmp to i32
-  %val = uitofp i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sbyte:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_cvt_f32_i32_e32 v0, v0
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sbyte(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = sext i8 %tmp to i32
-  %val = sitofp i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sshort:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, off, s[0:3], 0 offset:16
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_cvt_f32_i32_e32 v0, v0
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sshort(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  %tmp2 = sext i16 %tmp to i32
-  %val = sitofp i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ubyte_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ubyte_bitcast(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = zext i8 %tmp to i32
-  %val = bitcast i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ushort_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ushort_bitcast(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = zext i16 %tmp to i32
-  %val = bitcast i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sbyte_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sbyte_bitcast(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = sext i8 %tmp to i32
-  %val = bitcast i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sshort_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sshort_bitcast(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = sext i16 %tmp to i32
-  %val = bitcast i32 %tmp2 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ubyte_mul_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ubyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
-  %tmp2 = zext i8 %tmp to i32
-  %tmp3 = mul i32 %tmp2, 255
-  %val = bitcast i32 %tmp3 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_ushort_mul_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_ushort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
-  %tmp2 = zext i16 %tmp to i32
-  %tmp3 = mul i32 %tmp2, 255
-  %val = bitcast i32 %tmp3 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sbyte_mul_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sbyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
-  %tmp2 = sext i8 %tmp to i32
-  %tmp3 = mul i32 %tmp2, 255
-  %val = bitcast i32 %tmp3 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sshort_mul_bitcast:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sshort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
-main_body:
-  %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
-  %tmp2 = sext i16 %tmp to i32
-  %tmp3 = mul i32 %tmp2, 255
-  %val = bitcast i32 %tmp3 to float
-  ret float %val
-}
-
-;CHECK-LABEL: {{^}}buffer_load_sbyte_type_check:
-;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, off, s[0:3], 0 offset:8
-;CHECK-NEXT: s_waitcnt vmcnt(0)
-;CHECK-NEXT: v_bfe_i32 v{{[0-9]}}, v{{[0-9]}}, 0, 5
-;CHECK-NEXT: ; return to shader part epilog
-define amdgpu_ps float @buffer_load_sbyte_type_check(<4 x i32> inreg %rsrc) {
-main_body:
-  %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  %tmp2 = zext i8 %tmp to i32
-  %tmp3 = shl i32 %tmp2, 27
-  %tmp4 = ashr i32 %tmp3, 27
-  %val = bitcast i32 %tmp4 to float
-  ret float %val
-}
-
-; Make sure a frame index folding doesn't crash on a MUBUF not used
-; for stack access.
-
-; CHECK-LABEL: {{^}}no_fold_fi_imm_soffset:
-; CHECK: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}}
-; CHECK-NEXT: buffer_load_dword v0, [[FI]], s{{\[[0-9]+:[0-9]+\]}}, 0 idxen
-define amdgpu_ps float @no_fold_fi_imm_soffset(<4 x i32> inreg %rsrc) {
-  %alloca = alloca i32, addrspace(5)
-  %alloca.cast = ptrtoint ptr addrspace(5) %alloca to i32
-
-  %ret.val = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %alloca.cast, i32 0, i1 false, i1 false)
-  ret float %ret.val
-}
-
-; CHECK-LABEL: {{^}}no_fold_fi_reg_soffset:
-; CHECK-DAG: v_mov_b32_e32 v[[FI:[0-9]+]], 0{{$}}
-; CHECK-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s
-; CHECK: buffer_load_dword v0, v[[[FI]]:[[HI]]
-define amdgpu_ps float @no_fold_fi_reg_soffset(<4 x i32> inreg %rsrc, i32 inreg %soffset) {
-  %alloca = alloca i32, addrspace(5)
-  %alloca.cast = ptrtoint ptr addrspace(5) %alloca to i32
-
-  %ret.val = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %alloca.cast, i32 %soffset, i1 false, i1 false)
-  ret float %ret.val
-}
-
-declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #0
-declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
-declare i8 @llvm.amdgcn.buffer.load.i8(<4 x i32>, i32, i32, i1, i1) #0
-declare i16 @llvm.amdgcn.buffer.load.i16(<4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
-
-attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.dwordx3.ll
index 269956f948f5012916b4003aaa3e9410cb6cbd94..7723b565d962be5d854e29ecb157a4832ae1c160 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.dwordx3.ll
@@ -1,23 +1,5 @@
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
 
-;CHECK-LABEL: {{^}}buffer_store_format_immoffs_x3:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyz v[0:2], off, s[0:3], 0 offset:42
-define amdgpu_ps void @buffer_store_format_immoffs_x3(<4 x i32> inreg, <3 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v3f32(<3 x float> %1, <4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_immoffs_x3:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx3 v[0:2], off, s[0:3], 0 offset:42
-define amdgpu_ps void @buffer_store_immoffs_x3(<4 x i32> inreg, <3 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v3f32(<3 x float> %1, <4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret void
-}
-
 ;CHECK-LABEL: {{^}}raw_buffer_store_format_immoffs_x3:
 ;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyz v[0:2], off, s[0:3], 0 offset:42
@@ -72,8 +54,6 @@ main_body:
   ret void
 }
 
-declare void @llvm.amdgcn.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i1, i1) #0
 declare void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll
deleted file mode 100644
index a8cabdc9b03041754113d461527b6349b0437716..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll
+++ /dev/null
@@ -1,63 +0,0 @@
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,UNPACKED %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED %s
-
-; GCN-LABEL: {{^}}buffer_store_format_d16_x:
-; GCN: s_load_dword s[[LO:[0-9]+]]
-; GCN: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[LO]]
-; GCN: buffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
-define amdgpu_kernel void @buffer_store_format_d16_x(<4 x i32> %rsrc, [8 x i32], half %data, [8 x i32], i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.f16(half %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_format_d16_xy:
-
-; UNPACKED: s_load_dwordx2 s[[[S_DATA:[0-9]+]]:{{[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}}, 0x10
-; UNPACKED-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s[[S_DATA]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED:s[0-9]+]], s[[S_DATA]], 0xffff{{$}}
-; UNPACKED-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[MASKED]]
-; UNPACKED-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], [[SHR]]
-; UNPACKED: buffer_store_format_d16_xy v[[[V_LO]]:[[V_HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
-
-; PACKED: buffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
-define amdgpu_kernel void @buffer_store_format_d16_xy(<4 x i32> %rsrc, <2 x half> %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <3 x half> %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v3f16(<3 x half> %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_format_d16_xyzw:
-; GCN-DAG: s_load_dwordx2 s[[[S_DATA_0:[0-9]+]]:[[S_DATA_1:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x10
-
-; UNPACKED-DAG: s_lshr_b32 [[SHR0:s[0-9]+]], s[[S_DATA_0]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED0:s[0-9]+]], s[[S_DATA_0]], 0xffff{{$}}
-; UNPACKED-DAG: s_lshr_b32 [[SHR1:s[0-9]+]], s[[S_DATA_1]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED1:s[0-9]+]], s[[S_DATA_1]], 0xffff{{$}}
-
-; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
-; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-
-; UNPACKED: buffer_store_format_d16_xyzw v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
-
-; PACKED: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
-; PACKED: v_mov_b32_e32 v[[HI:[0-9]+]], s[[S_DATA_1]]
-
-; PACKED: buffer_store_format_d16_xyzw v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
-define amdgpu_kernel void @buffer_store_format_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.buffer.store.format.f16(half, <4 x i32>, i32, i32, i1, i1)
-declare void @llvm.amdgcn.buffer.store.format.v2f16(<2 x half>, <4 x i32>, i32, i32, i1, i1)
-declare void @llvm.amdgcn.buffer.store.format.v3f16(<3 x half>, <4 x i32>, i32, i32, i1, i1)
-declare void @llvm.amdgcn.buffer.store.format.v4f16(<4 x half>, <4 x i32>, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
deleted file mode 100644
index 41e2b4d0e55125fef5e4988a8e016a9158c9d381..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
+++ /dev/null
@@ -1,104 +0,0 @@
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=VERDE %s
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
-
-;CHECK-LABEL: {{^}}buffer_store:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], off, s[0:3], 0
-;CHECK: buffer_store_format_xyzw v[4:7], off, s[0:3], 0 glc
-;CHECK: buffer_store_format_xyzw v[8:11], off, s[0:3], 0 slc
-define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 0, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %2, <4 x i32> %0, i32 0, i32 0, i1 1, i1 0)
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %3, <4 x i32> %0, i32 0, i32 0, i1 0, i1 1)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_immoffs:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], off, s[0:3], 0 offset:42
-define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_idx:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_ofs:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 offen
-define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 %2, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_both:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 idxen offen
-define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 %3, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_both_reversed:
-;CHECK: v_mov_b32_e32 v6, v4
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], v[5:6], s[0:3], 0 idxen offen
-define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 %3, i32 %2, i1 0, i1 0)
-  ret void
-}
-
-; Ideally, the register allocator would avoid the wait here
-;
-;CHECK-LABEL: {{^}}buffer_store_wait:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
-;VERDE: s_waitcnt expcnt(0)
-;CHECK: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_store_format_xyzw v[0:3], v6, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i1 0, i1 0)
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %3, i32 0, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %data, <4 x i32> %0, i32 %4, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x1:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_x v0, v1, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.f32(float %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x2:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_format_xy v[0:1], v2, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.format.v2f32(<2 x float> %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.buffer.store.format.f32(float, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.format.v2f32(<2 x float>, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll
deleted file mode 100644
index 8b18848a62792c8549b53296170ce4947512e09f..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll
+++ /dev/null
@@ -1,268 +0,0 @@
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=VERDE %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
-
-;CHECK-LABEL: {{^}}buffer_store:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-;CHECK: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 glc
-;CHECK: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 slc
-define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 0, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %2, <4 x i32> %0, i32 0, i32 0, i1 1, i1 0)
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %3, <4 x i32> %0, i32 0, i32 0, i1 0, i1 1)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_immoffs:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:42
-define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_idx:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_ofs:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
-define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 %2, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_both:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 idxen offen
-define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 %3, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_both_reversed:
-;CHECK: v_mov_b32_e32 v6, v4
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], v[5:6], s[0:3], 0 idxen offen
-define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %3, i32 %2, i1 0, i1 0)
-  ret void
-}
-
-; Ideally, the register allocator would avoid the wait here
-;
-;CHECK-LABEL: {{^}}buffer_store_wait:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen
-;VERDE: s_waitcnt expcnt(0)
-;CHECK: buffer_load_dwordx4 v[0:3], v5, s[0:3], 0 idxen
-;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_store_dwordx4 v[0:3], v6, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32, i32) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i1 0, i1 0)
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 %3, i32 0, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %data, <4 x i32> %0, i32 %4, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x1:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dword v0, v1, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %index) {
-main_body:
-  call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x2:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 idxen
-define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %index) #0 {
-main_body:
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %data, <4 x i32> %rsrc, i32 %index, i32 0, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
-define amdgpu_ps void @buffer_store_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 8
-  %a3 = add i32 %a, 12
-  %a4 = add i32 %a, 16
-  %a5 = add i32 %a, 28
-  %a6 = add i32 %a, 32
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 %a3, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 %a4, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 %a5, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 %a6, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged_glc_slc:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
-;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:12 glc{{$}}
-;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 glc slc{{$}}
-define amdgpu_ps void @buffer_store_x1_offen_merged_glc_slc(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 8
-  %a3 = add i32 %a, 12
-  %a4 = add i32 %a, 16
-  %a5 = add i32 %a, 28
-  %a6 = add i32 %a, 32
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 %a3, i1 1, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 %a4, i1 1, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 %a5, i1 1, i1 1)
-  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 %a6, i1 1, i1 1)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x2_offen_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-define amdgpu_ps void @buffer_store_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a, <2 x float> %v1, <2 x float> %v2) {
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 12
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offen_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
-define amdgpu_ps void @buffer_store_x3_offen_merged(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3) {
-  %a1 = add i32 %a, 28
-  %a2 = add i32 %a, 32
-  %a3 = add i32 %a, 36
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 %a3, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offen_merged2:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-define amdgpu_ps void @buffer_store_x3_offen_merged2(<4 x i32> inreg %rsrc, i32 %a, <2 x float> %v1, float %v2) {
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 12
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offen_merged3:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-define amdgpu_ps void @buffer_store_x3_offen_merged3(<4 x i32> inreg %rsrc, i32 %a, float %v1, <2 x float> %v2) {
-  %a1 = add i32 %a, 4
-  %a2 = add i32 %a, 8
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x1_offset_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
-define amdgpu_ps void @buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 28, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 32, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x2_offset_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-define amdgpu_ps void @buffer_store_x2_offset_merged(<4 x i32> inreg %rsrc, <2 x float> %v1, <2 x float> %v2) {
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offset_merged:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-define amdgpu_ps void @buffer_store_x3_offset_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3) {
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offset_merged2:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
-define amdgpu_ps void @buffer_store_x3_offset_merged2(<4 x i32> inreg %rsrc, float %v1, <2 x float> %v2) {
-  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_x3_offset_merged3:
-;CHECK-NOT: s_waitcnt
-;CHECK-DAG: buffer_store_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:8
-define amdgpu_ps void @buffer_store_x3_offset_merged3(<4 x i32> inreg %rsrc, <2 x float> %v1, float %v2) {
-  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_byte:
-;CHECK-NOT: s_waitcnt
-;CHECK-NEXT: %bb.
-;CHECK: buffer_store_byte v{{[0-9]}}, off, s[0:3], 0 offset:8
-define amdgpu_ps void @buffer_store_byte(<4 x i32> inreg %rsrc, float %v1) {
-main_body:
-  %v2 = fptoui float %v1 to i32
-  %v3 = trunc i32 %v2 to i8
-  call void @llvm.amdgcn.buffer.store.i8(i8 %v3, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
-  ret void
-}
-
-;CHECK-LABEL: {{^}}buffer_store_short:
-;CHECK-NOT: s_waitcnt
-;CHECK-NEXT: %bb.
-;CHECK: buffer_store_short v{{[0-9]}}, off, s[0:3], 0 offset:16
-define amdgpu_ps void @buffer_store_short(<4 x i32> inreg %rsrc, float %v1) {
-main_body:
-  %v2 = fptoui float %v1 to i32
-  %v3 = trunc i32 %v2 to i16
-  call void @llvm.amdgcn.buffer.store.i16(i16 %v3, <4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.i8(i8, <4 x i32>, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.buffer.store.i16(i16, <4 x i32>, i32, i32, i1, i1) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll
deleted file mode 100644
index bd1888b6965a149aecfc4caff86987a1d12f6ad2..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=fiji -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-
-declare void @llvm.amdgcn.buffer.wbinvl1() #0
-
-; GCN-LABEL: {{^}}test_buffer_wbinvl1:
-; GCN-NEXT: ; %bb.0:
-; SI-NEXT: buffer_wbinvl1 ; encoding: [0x00,0x00,0xc4,0xe1,0x00,0x00,0x00,0x00]
-; VI-NEXT: buffer_wbinvl1 ; encoding: [0x00,0x00,0xf8,0xe0,0x00,0x00,0x00,0x00]
-; GCN-NEXT: s_endpgm
-define amdgpu_kernel void @test_buffer_wbinvl1() #0 {
-  call void @llvm.amdgcn.buffer.wbinvl1()
-  ret void
-}
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll
deleted file mode 100644
index b937c42e14ed5a7ac53c33e1237874772ce18a32..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -show-mc-encoding < %s | FileCheck -check-prefix=SI %s
-
-declare void @llvm.amdgcn.buffer.wbinvl1.sc() #0
-
-; SI-LABEL: {{^}}test_buffer_wbinvl1_sc:
-; SI-NEXT: ; %bb.0:
-; SI-NEXT: buffer_wbinvl1_sc ; encoding: [0x00,0x00,0xc0,0xe1,0x00,0x00,0x00,0x00]
-; SI-NEXT: s_endpgm
-define amdgpu_kernel void @test_buffer_wbinvl1_sc() #0 {
-  call void @llvm.amdgcn.buffer.wbinvl1.sc()
-  ret void
-}
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll
deleted file mode 100644
index 64ab8ecefd490902edd32cfa39557768d225f336..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: llc -mtriple=amdgcn -mcpu=bonaire -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-
-declare void @llvm.amdgcn.buffer.wbinvl1.vol() #0
-
-; GCN-LABEL: {{^}}test_buffer_wbinvl1_vol:
-; GCN-NEXT: ; %bb.0:
-; CI: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xc0,0xe1,0x00,0x00,0x00,0x00]
-; VI: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
-; GCN: _store_byte
-; GCN-NEXT: s_endpgm
-define amdgpu_kernel void @test_buffer_wbinvl1_vol(ptr addrspace(1) %ptr) #0 {
-  call void @llvm.amdgcn.buffer.wbinvl1.vol()
-; This used to crash in hazard recognizer
-  store i8 0, ptr addrspace(1) %ptr, align 1
-  ret void
-}
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
index 98ed437581e1af138c51f105611053f8769adfed..47b7658f50cc5a7ba9f9e703d272ab8cbc6084d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
@@ -313,7 +313,7 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float> %vdata, i3
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
   call void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 %vindex.1, i32 0, i32 0, i32 63, i32 0)
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %vindex.2, i32 0, i1 0, i1 0)
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %0, i32 %vindex.2, i32 0, i32 0, i32 0)
   %data.i = bitcast <4 x float> %data to <4 x i32>
   call void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32> %data.i, <4 x i32> %0, i32 %vindex.3, i32 0, i32 0, i32 46, i32 0)
   ret void
@@ -620,7 +620,7 @@ declare void @llvm.amdgcn.struct.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32
 declare void @llvm.amdgcn.struct.tbuffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32, i32) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
deleted file mode 100644
index 06d66ce9db37d2da74c82e515b741a2e6cfce55b..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=gfx810 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
-
-; GCN-LABEL: {{^}}tbuffer_load_d16_x:
-; GCN: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call half @llvm.amdgcn.tbuffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
-  ret half %data
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_d16_xy:
-; UNPACKED: tbuffer_load_format_d16_xy v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[FULL]]
-define amdgpu_ps half @tbuffer_load_d16_xy(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <2 x half> @llvm.amdgcn.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
-  %elt = extractelement <2 x half> %data, i32 1
-  ret half %elt
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_d16_xyz:
-; UNPACKED: tbuffer_load_format_d16_xyz v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: tbuffer_load_format_d16_xyz v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; PACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-define amdgpu_ps half @tbuffer_load_d16_xyz(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <3 x half> @llvm.amdgcn.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
-  %elt = extractelement <3 x half> %data, i32 2
-  ret half %elt
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_d16_xyzw:
-; UNPACKED: tbuffer_load_format_d16_xyzw v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
-
-; PACKED: tbuffer_load_format_d16_xyzw v[{{[0-9]+}}:[[HI:[0-9]+]]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[HI]]
-define amdgpu_ps half @tbuffer_load_d16_xyzw(<4 x i32> inreg %rsrc) {
-main_body:
-  %data = call <4 x half> @llvm.amdgcn.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
-  %elt = extractelement <4 x half> %data, i32 3
-  ret half %elt
-}
-
-declare half @llvm.amdgcn.tbuffer.load.f16(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <2 x half> @llvm.amdgcn.tbuffer.load.v2f16(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <3 x half> @llvm.amdgcn.tbuffer.load.v3f16(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <4 x half> @llvm.amdgcn.tbuffer.load.v4f16(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
index de929c2893baf9e1017b23d7cfc606d11a8291b6..c89c5c5599334beb134e030a76ff3deab024372a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
@@ -23,18 +23,5 @@ main_body:
     ret <3 x float> %vdata.f
 }
 
-
-; GCN-LABEL: {{^}}tbuffer_load_format_immoffs_x3:
-; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
-; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
-define amdgpu_vs <3 x float> @tbuffer_load_format_immoffs_x3(<4 x i32> inreg) {
-main_body:
-    %vdata   = call <3 x i32> @llvm.amdgcn.tbuffer.load.v3i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <3 x i32> %vdata to <3 x float>
-    ret <3 x float> %vdata.f
-}
-
 declare <3 x i32> @llvm.amdgcn.raw.tbuffer.load.v3i32(<4 x i32>, i32, i32, i32, i32)
 declare <3 x i32> @llvm.amdgcn.struct.tbuffer.load.v3i32(<4 x i32>, i32, i32, i32, i32, i32)
-declare <3 x i32> @llvm.amdgcn.tbuffer.load.v3i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
deleted file mode 100644
index eb0dc37849b0f49b386e802226370f75cd1f19cd..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
+++ /dev/null
@@ -1,109 +0,0 @@
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=GCN %s
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
-
-; GCN-LABEL: {{^}}tbuffer_load:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] glc
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] slc
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
-; GCN: s_waitcnt
-define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>, <4 x float>} @tbuffer_load(<4 x i32> inreg) {
-main_body:
-    %vdata     = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
-    %vdata_glc = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 15, i32 3, i1 1, i1 0)
-    %vdata_slc = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 1)
-    %vdata_f32 = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
-    %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
-    %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
-    %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
-    %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
-    %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
-    %r3 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r2, <4 x float> %vdata_f32, 3
-    ret {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r3
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_immoffs:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
-define amdgpu_vs <4 x float> @tbuffer_load_immoffs(<4 x i32> inreg) {
-main_body:
-    %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <4 x i32> %vdata to <4 x float>
-    ret <4 x float> %vdata.f
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_immoffs_large
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] offset:4095
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_SSCALED] offset:73
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] offset:1
-; GCN: s_waitcnt
-define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_large(<4 x i32> inreg, i32 inreg %soffs) {
-    %vdata     = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 61, i32 4095, i32 15, i32 2, i1 0, i1 0)
-    %vdata_glc = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 %soffs, i32 73, i32 14, i32 3, i1 0, i1 0)
-    %vdata_slc = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 %soffs, i32 1, i32 13, i32 4, i1 0, i1 0)
-    %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
-    %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
-    %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
-    %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
-    %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
-    ret {<4 x float>, <4 x float>, <4 x float>} %r2
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_idx:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen
-define amdgpu_vs <4 x float> @tbuffer_load_idx(<4 x i32> inreg, i32 %vindex) {
-main_body:
-    %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 0, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <4 x i32> %vdata to <4 x float>
-    ret <4 x float> %vdata.f
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_ofs:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen
-define amdgpu_vs <4 x float> @tbuffer_load_ofs(<4 x i32> inreg, i32 %voffs) {
-main_body:
-    %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 %voffs, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <4 x i32> %vdata to <4 x float>
-    ret <4 x float> %vdata.f
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_ofs_imm:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen offset:52
-define amdgpu_vs <4 x float> @tbuffer_load_ofs_imm(<4 x i32> inreg, i32 %voffs) {
-main_body:
-    %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 %voffs, i32 0, i32 52, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <4 x i32> %vdata to <4 x float>
-    ret <4 x float> %vdata.f
-}
-
-; GCN-LABEL: {{^}}tbuffer_load_both:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offen
-define amdgpu_vs <4 x float> @tbuffer_load_both(<4 x i32> inreg, i32 %vindex, i32 %voffs) {
-main_body:
-    %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 %voffs, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <4 x i32> %vdata to <4 x float>
-    ret <4 x float> %vdata.f
-}
-
-
-; GCN-LABEL: {{^}}buffer_load_xy:
-; GCN: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
-define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
-    %vdata = call <2 x i32> @llvm.amdgcn.tbuffer.load.v2i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 13, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast <2 x i32> %vdata to <2 x float>
-    ret <2 x float> %vdata.f
-}
-
-; GCN-LABEL: {{^}}buffer_load_x:
-; GCN: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
-define amdgpu_vs float @buffer_load_x(<4 x i32> inreg %rsrc) {
-    %vdata = call i32 @llvm.amdgcn.tbuffer.load.i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 13, i32 4, i1 0, i1 0)
-    %vdata.f = bitcast i32 %vdata to float
-    ret float %vdata.f
-}
-
-declare i32 @llvm.amdgcn.tbuffer.load.i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <2 x i32> @llvm.amdgcn.tbuffer.load.v2i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
deleted file mode 100644
index 16d7de698aa03f627d0ed812fefbfadcc4173405..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
+++ /dev/null
@@ -1,76 +0,0 @@
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,UNPACKED %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,PACKED %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,PACKED %s
-
-
-; GCN-LABEL: {{^}}tbuffer_store_d16_x:
-; GCN: s_load_dword s[[S_LO:[0-9]+]]
-; GCN: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[S_LO]]
-; GCN: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, [8 x i32], half %data, [8 x i32], i32 %vindex) {
-main_body:
-  call void @llvm.amdgcn.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}tbuffer_store_d16_xy:
-; GCN: s_load_dwordx2 s[[[S_DATA:[0-9]+]]:{{[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}}, 0x10
-; UNPACKED-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s[[S_DATA]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED:s[0-9]+]], s[[S_DATA]], 0xffff{{$}}
-; UNPACKED-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[MASKED]]
-; UNPACKED-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], [[SHR]]
-; UNPACKED: tbuffer_store_format_d16_xy v[[[V_LO]]:[[V_HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-
-; PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_kernel void @tbuffer_store_d16_xy(<4 x i32> %rsrc, <2 x half> %data, i32 %vindex) {
-main_body:
-  call void @llvm.amdgcn.tbuffer.store.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}tbuffer_store_d16_xyz:
-; GCN-DAG: s_load_dwordx2 s[[[S_DATA_0:[0-9]+]]:[[S_DATA_1:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x10
-
-; UNPACKED-DAG: s_lshr_b32 [[SHR0:s[0-9]+]], s[[S_DATA_0]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED0:s[0-9]+]], s[[S_DATA_0]], 0xffff{{$}}
-; UNPACKED-DAG: s_and_b32 [[SHR1:s[0-9]+]], s[[S_DATA_1]], 0xffff{{$}}
-
-; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
-; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-; UNPACKED: tbuffer_store_format_d16_xyz v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-
-; PACKED-DAG: s_and_b32 [[SHR0:s[0-9]+]], s[[S_DATA_1]], 0xffff{{$}}
-; PACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
-; PACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR0]]
-; PACKED: tbuffer_store_format_d16_xyz v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <3 x half> %data, i32 %vindex) {
-main_body:
-  call void @llvm.amdgcn.tbuffer.store.v3f16(<3 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}tbuffer_store_d16_xyzw:
-; GCN-DAG: s_load_dwordx2 s[[[S_DATA_0:[0-9]+]]:[[S_DATA_1:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x10
-
-; UNPACKED-DAG: s_lshr_b32 [[SHR0:s[0-9]+]], s[[S_DATA_0]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED0:s[0-9]+]], s[[S_DATA_0]], 0xffff{{$}}
-; UNPACKED-DAG: s_lshr_b32 [[SHR1:s[0-9]+]], s[[S_DATA_1]], 16
-; UNPACKED-DAG: s_and_b32 [[MASKED1:s[0-9]+]], s[[S_DATA_1]], 0xffff{{$}}
-
-; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
-; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-; UNPACKED: tbuffer_store_format_d16_xyzw v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-
-; PACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
-; PACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s[[S_DATA_1]]
-; PACKED: tbuffer_store_format_d16_xyzw v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %vindex) {
-main_body:
-  call void @llvm.amdgcn.tbuffer.store.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.tbuffer.store.f16(half, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare void @llvm.amdgcn.tbuffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare void @llvm.amdgcn.tbuffer.store.v3f16(<3 x half>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare void @llvm.amdgcn.tbuffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
index 83b5b0c65c47e00a95718df016cb7c9d0ab1de8a..d5cbaddff020b0c61b18d8e086e95df284b5ef81 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
@@ -20,16 +20,6 @@ main_body:
   ret void
 }
 
-; GCN-LABEL: {{^}}tbuffer_store_immoffs_x3:
-; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
-define amdgpu_ps void @tbuffer_store_immoffs_x3(<4 x i32> inreg, <3 x float>) {
-main_body:
-  %in1 = bitcast <3 x float> %1 to <3 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v3i32(<3 x i32> %in1, <4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 5, i32 7, i1 0, i1 0)
-  ret void
-}
-
 declare void @llvm.amdgcn.raw.tbuffer.store.v3i32(<3 x i32>, <4 x i32>, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.struct.tbuffer.store.v3i32(<3 x i32>, <4 x i32>, i32, i32, i32, i32, i32) #0
-declare void @llvm.amdgcn.tbuffer.store.v3i32(<3 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
deleted file mode 100644
index b3a135a35a6526dce30fa5c744bd32892d5a553b..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
+++ /dev/null
@@ -1,110 +0,0 @@
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefixes=GCN,VERDE %s
-;RUN: llc -global-isel=0 < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
-
-; GCN-LABEL: {{^}}tbuffer_store:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16_16_16,BUF_NUM_FORMAT_USCALED]
-; GCN: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_SSCALED] glc
-; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] slc
-; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
-define amdgpu_ps void @tbuffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
-main_body:
-  %in1 = bitcast <4 x float> %1 to <4 x i32>
-  %in2 = bitcast <4 x float> %2 to <4 x i32>
-  %in3 = bitcast <4 x float> %3 to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 12, i32 2, i1 0, i1 0)
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in2, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 13, i32 3, i1 1, i1 0)
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in3, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 14, i32 4, i1 0, i1 1)
-  call void @llvm.amdgcn.tbuffer.store.v4f32(<4 x float> %3, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}tbuffer_store_immoffs:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
-define amdgpu_ps void @tbuffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
-main_body:
-  %in1 = bitcast <4 x float> %1 to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 5, i32 7, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}tbuffer_store_scalar_and_imm_offs:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], {{s[0-9]+}} format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
-define amdgpu_ps void @tbuffer_store_scalar_and_imm_offs(<4 x i32> inreg, <4 x float> %vdata, i32 inreg %soffset) {
-main_body:
-  %in1 = bitcast <4 x float> %vdata to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 0, i32 0, i32 %soffset, i32 42, i32 5, i32 7, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_idx:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex) {
-main_body:
-  %in1 = bitcast <4 x float> %vdata to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 %vindex, i32 0, i32 0, i32 0, i32 15, i32 2, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_ofs:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_8_8,BUF_NUM_FORMAT_FLOAT] offen
-define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float> %vdata, i32 %voffset) {
-main_body:
-  %in1 = bitcast <4 x float> %vdata to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 0, i32 %voffset, i32 0, i32 0, i32 3, i32 7, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_both:
-; GCN: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_UINT] idxen offen
-define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex, i32 %voffset) {
-main_body:
-  %in1 = bitcast <4 x float> %vdata to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 %vindex, i32 %voffset, i32 0, i32 0, i32 6, i32 4, i1 0, i1 0)
-  ret void
-}
-
-; Ideally, the register allocator would avoid the wait here
-;
-; GCN-LABEL: {{^}}buffer_store_wait:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] idxen
-; VERDE: s_waitcnt expcnt(0)
-; GCN: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen
-; GCN: s_waitcnt vmcnt(0)
-; GCN: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex.1, i32 %vindex.2, i32 %vindex.3) {
-main_body:
-  %in1 = bitcast <4 x float> %vdata to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %in1, <4 x i32> %0, i32 %vindex.1, i32 0, i32 0, i32 0, i32 15, i32 3, i1 0, i1 0)
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 %vindex.2, i32 0, i1 0, i1 0)
-  %data.i = bitcast <4 x float> %data to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %data.i, <4 x i32> %0, i32 %vindex.3, i32 0, i32 0, i32 0, i32 14, i32 2, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_x1:
-; GCN: tbuffer_store_format_x v0, v1, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen
-define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
-main_body:
-  %data.i = bitcast float %data to i32
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %data.i, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 13, i32 7, i1 0, i1 0)
-  ret void
-}
-
-; GCN-LABEL: {{^}}buffer_store_x2:
-; GCN: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
-define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %vindex) {
-main_body:
-  %data.i = bitcast <2 x float> %data to <2 x i32>
-  call void @llvm.amdgcn.tbuffer.store.v2i32(<2 x i32> %data.i, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
-  ret void
-}
-
-declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.tbuffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
-declare void @llvm.amdgcn.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readonly }
-
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index 7d7a462597104fb6c23eb83215b9f993f2a5bccb..fa7ee9e8d28ff63d732a3eeaba114f0a2c05d33e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -554,28 +554,14 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; GFX8-NEXT:    v_max_f16_e32 v4, v3, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX8-NEXT:    v_max_f16_e32 v3, v0, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -654,46 +640,24 @@ define <2 x half> @v_maximum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) {
 ; GFX7-LABEL: v_maximum_v2f16__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f16__nnan:
@@ -759,13 +723,11 @@ define <2 x half> @v_maximum_v2f16__nsz(<2 x half> %src0, <2 x half> %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; GFX8-NEXT:    v_max_f16_e32 v4, v3, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX8-NEXT:    v_max_f16_e32 v3, v0, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
@@ -847,34 +809,24 @@ define <2 x half> @v_maximum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1)
 ; GFX7-LABEL: v_maximum_v2f16__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f16__nnan_nsz:
@@ -948,31 +900,15 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX8-NEXT:    s_lshr_b32 s6, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, s7, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
+; GFX8-NEXT:    v_max_f16_e32 v1, s7, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s7, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_gt_f16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX8-NEXT:    v_max_f16_e32 v3, s4, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
@@ -1216,28 +1152,21 @@ define <3 x half> @v_maximum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) {
 ; GFX7-LABEL: v_maximum_v3f16__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
-; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16__nnan:
@@ -1427,28 +1356,21 @@ define <3 x half> @v_maximum_v3f16__nnan_nsz(<3 x half> %src0, <3 x half> %src1)
 ; GFX7-LABEL: v_maximum_v3f16__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
-; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16__nnan_nsz:
@@ -1671,35 +1593,26 @@ define <4 x half> @v_maximum_v4f16__nnan(<4 x half> %src0, <4 x half> %src1) {
 ; GFX7-LABEL: v_maximum_v4f16__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
-; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16__nnan:
@@ -1924,35 +1837,26 @@ define <4 x half> @v_maximum_v4f16__nnan_nsz(<4 x half> %src0, <4 x half> %src1)
 ; GFX7-LABEL: v_maximum_v4f16__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
-; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16__nnan_nsz:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
index 7c5bc7da4df2e105147fc25784aadfcfb6c08550..f4aa40dbd9bcd7801140b8f2b22487edb729befb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
@@ -495,167 +495,73 @@ define <2 x float> @v_maximum_v2f32(<2 x float> %src0, <2 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v2f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v1, v3
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX8-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX940-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v2, v1, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v5, v1, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v4, v0, v2 :: v_dual_max_f32 v5, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f32:
@@ -676,136 +582,42 @@ define <2 x float> @v_maximum_v2f32__nnan(<2 x float> %src0, <2 x float> %src1)
 ; GFX7-LABEL: v_maximum_v2f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v0, v2
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v1, v3
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f32__nnan:
@@ -826,11 +638,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v2f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v1, v3
+; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -838,13 +650,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX8-LABEL: v_maximum_v2f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -852,13 +662,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX9-LABEL: v_maximum_v2f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -866,16 +674,12 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX940-LABEL: v_maximum_v2f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX940-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_max_f32_e32 v2, v1, v3
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
@@ -884,11 +688,9 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX10-LABEL: v_maximum_v2f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
+; GFX10-NEXT:    v_max_f32_e32 v5, v1, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
@@ -897,12 +699,9 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX11-LABEL: v_maximum_v2f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v4, v0, v2 :: v_dual_max_f32 v5, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
@@ -926,55 +725,42 @@ define <2 x float> @v_maximum_v2f32__nnan_nsz(<2 x float> %src0, <2 x float> %sr
 ; GFX7-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v0, v2
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v1, v3
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f32__nnan_nsz:
@@ -996,28 +782,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, s5, v0
+; GFX7-NEXT:    v_max_f32_e32 v1, s5, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, s4, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_max_f32_e32 v3, s4, v0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v3, s4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use v[0:1]
 ; GFX7-NEXT:    ;;#ASMEND
@@ -1027,30 +799,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s7
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
+; GFX8-NEXT:    v_max_f32_e32 v1, s5, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v3, s4, v0
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
@@ -1060,30 +816,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
-; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, s5, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
+; GFX9-NEXT:    v_max_f32_e32 v1, s5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, s4, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v[0:1]
 ; GFX9-NEXT:    ;;#ASMEND
@@ -1093,40 +833,15 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s3
-; GFX940-NEXT:    v_mov_b32_e32 v1, s1
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, s1, v0
-; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v1, s1, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s1, v0
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v0
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v3, s0, v0
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
@@ -1135,28 +850,12 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX10-LABEL: s_maximum_v2f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s5
-; GFX10-NEXT:    v_cmp_gt_f32_e64 vcc_lo, s5, s7
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    v_cmp_class_f32_e64 s8, s5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, s7, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e64 vcc_lo, s4, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s6, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e64 v0, s5, s7
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s5, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX10-NEXT:    v_max_f32_e64 v2, s4, s6
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s4, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s5, s8
-; GFX10-NEXT:    v_cmp_class_f32_e64 s5, s4, 64
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v2, s4, s5
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s7, 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s7, s4
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s6, s4
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v[0:1]
 ; GFX10-NEXT:    ;;#ASMEND
@@ -1165,32 +864,13 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX11-LABEL: s_maximum_v2f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT:    v_cmp_gt_f32_e64 vcc_lo, s1, s3
-; GFX11-NEXT:    v_cmp_class_f32_e64 s4, s1, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, s3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e64 vcc_lo, s0, s2
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_max_f32_e64 v0, s1, s3
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s1, s3
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX11-NEXT:    v_max_f32_e64 v2, s0, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s1, s4
-; GFX11-NEXT:    v_cmp_class_f32_e64 s1, s0, 64
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v2, s0, s1
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s3, 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s2, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s2, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v[0:1]
 ; GFX11-NEXT:    ;;#ASMEND
@@ -1218,227 +898,92 @@ define <3 x float> @v_maximum_v3f32(<3 x float> %src0, <3 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v3f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v6, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v1, v4
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v2, v5
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX8-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX8-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
+; GFX940-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v3, v1, v4
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX940-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v7, v1, v4
+; GFX10-NEXT:    v_max_f32_e32 v8, v2, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v6, v0, v3 :: v_dual_max_f32 v7, v1, v4
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f32:
@@ -1460,184 +1005,48 @@ define <3 x float> @v_maximum_v3f32__nnan(<3 x float> %src0, <3 x float> %src1)
 ; GFX7-LABEL: v_maximum_v3f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v6, v0, v3
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v1, v4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v2, v5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX8-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX9-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX940-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX10-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v3 :: v_dual_max_f32 v1, v1, v4
+; GFX11-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f32__nnan:
@@ -1659,14 +1068,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v3f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v6, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v1, v4
+; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v2, v5
+; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -1674,17 +1083,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX8-LABEL: v_maximum_v3f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX8-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1692,17 +1098,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX9-LABEL: v_maximum_v3f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1710,22 +1113,16 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX940-LABEL: v_maximum_v3f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
+; GFX940-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_max_f32_e32 v3, v1, v4
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
@@ -1734,13 +1131,10 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX10-LABEL: v_maximum_v3f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
+; GFX10-NEXT:    v_max_f32_e32 v7, v1, v4
+; GFX10-NEXT:    v_max_f32_e32 v8, v2, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
@@ -1751,17 +1145,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX11-LABEL: v_maximum_v3f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v6, v0, v3 :: v_dual_max_f32 v7, v1, v4
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1784,67 +1175,48 @@ define <3 x float> @v_maximum_v3f32__nnan_nsz(<3 x float> %src0, <3 x float> %sr
 ; GFX7-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v0, v3
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v1, v4
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v2, v5
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX8-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX9-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX940-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v3
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v4
+; GFX10-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v3 :: v_dual_max_f32 v1, v1, v4
+; GFX11-NEXT:    v_max_f32_e32 v2, v2, v5
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f32__nnan_nsz:
@@ -1866,292 +1238,111 @@ define <4 x float> @v_maximum_v4f32(<4 x float> %src0, <4 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v4f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v1, v5
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v2, v6
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v3, v7
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
+; GFX940-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v4, v1, v5
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX940-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX940-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v9, v1, v5
+; GFX10-NEXT:    v_max_f32_e32 v4, v2, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v3, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v8, v0, v4 :: v_dual_max_f32 v9, v1, v5
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v4, v2, v6
+; GFX11-NEXT:    v_dual_max_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f32:
@@ -2174,236 +1365,53 @@ define <4 x float> @v_maximum_v4f32__nnan(<4 x float> %src0, <4 x float> %src1)
 ; GFX7-LABEL: v_maximum_v4f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v0, v4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v1, v5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v2, v6
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v3, v7
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX8-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX8-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX9-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX9-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX940-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX940-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX10-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX10-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v7, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v1, v1, v5
+; GFX11-NEXT:    v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f32__nnan:
@@ -2426,17 +1434,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v4f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v4, v3, v7
+; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -2444,21 +1452,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX8-LABEL: v_maximum_v4f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX8-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -2466,21 +1470,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX9-LABEL: v_maximum_v4f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2488,28 +1488,20 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX940-LABEL: v_maximum_v4f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
+; GFX940-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_max_f32_e32 v4, v1, v5
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX940-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
@@ -2518,44 +1510,35 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX10-LABEL: v_maximum_v4f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
+; GFX10-NEXT:    v_max_f32_e32 v9, v1, v5
+; GFX10-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v7, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v3, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v8, v0, v4 :: v_dual_max_f32 v9, v1, v5
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v7, v3, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v4, v2, v6
+; GFX11-NEXT:    v_dual_max_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f32__nsz:
@@ -2578,79 +1561,53 @@ define <4 x float> @v_maximum_v4f32__nnan_nsz(<4 x float> %src0, <4 x float> %sr
 ; GFX7-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v0, v4
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v1, v5
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v2, v6
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v3, v7
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX8-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX8-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX9-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX9-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX940-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX940-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX940-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX10-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX10-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX10-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v1, v1, v5
+; GFX11-NEXT:    v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f32__nnan_nsz:
@@ -2673,551 +1630,185 @@ define <8 x float> @v_maximum_v8f32(<8 x float> %src0, <8 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v8f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v0, v8
+; GFX7-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v1, v9
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v1, v9
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v2, v10
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v2, v10
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v3, v11
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v3, v11
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v4, v12
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v4, v12
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v5, v13
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v5, v13
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v6, v14
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v6, v14
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v8, v7, v15
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX7-NEXT:    v_max_f32_e32 v8, v7, v15
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v8f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v1, v9
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v2, v10
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v3, v11
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v4, v12
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v5, v13
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v6, v14
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX8-NEXT:    v_max_f32_e32 v8, v7, v15
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v8f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v1, v9
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v2, v10
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v3, v11
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v4, v12
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v5, v13
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v6, v14
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX9-NEXT:    v_max_f32_e32 v8, v7, v15
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v8f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v8
+; GFX940-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX940-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v1, v9
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v10
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v2, v10
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v11
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v3, v11
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v12
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v4, v12
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v13
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v5, v13
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v14
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v6, v14
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v15
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX940-NEXT:    v_max_f32_e32 v8, v7, v15
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v8f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v17, v1, v9
+; GFX10-NEXT:    v_max_f32_e32 v8, v2, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v12, v4, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v9, v3, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v10, v7, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v4, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v9, v5, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v8, v6, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v8f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v16, v0, v8 :: v_dual_max_f32 v17, v1, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v12, v4, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v9, v3, v11 :: v_dual_max_f32 v8, v2, v10
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v10, v7, v15
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v8, v4, v12 :: v_dual_cndmask_b32 v3, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v9, v5, v13 :: v_dual_cndmask_b32 v4, 0x7fc00000, v8
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_max_f32 v8, v6, v14 :: v_dual_cndmask_b32 v5, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v8f32:
@@ -3244,1071 +1835,371 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v16f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v32, v0, v16
-; GFX7-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v16, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v1, v17
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v16
+; GFX7-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX7-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX7-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v2, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v3, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v4, v20
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v20, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v5, v21
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v21, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v6, v22
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v22, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v7, v23
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v23, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v8, v24
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v24, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v9, v25
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v25, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v10, v26
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v26, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v11, v27
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v27, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v12, v28
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v28, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v13, v29
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v29, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v14, v30
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v30, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v18
+; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX7-NEXT:    v_max_f32_e32 v18, v13, v29
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX7-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v19
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX7-NEXT:    v_max_f32_e32 v4, v4, v20
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX7-NEXT:    v_max_f32_e32 v5, v5, v21
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX7-NEXT:    v_max_f32_e32 v6, v6, v22
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX7-NEXT:    v_max_f32_e32 v7, v7, v23
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX7-NEXT:    v_max_f32_e32 v8, v8, v24
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX7-NEXT:    v_max_f32_e32 v9, v9, v25
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX7-NEXT:    v_max_f32_e32 v10, v10, v26
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX7-NEXT:    v_max_f32_e32 v11, v11, v27
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX7-NEXT:    v_max_f32_e32 v12, v12, v28
+; GFX7-NEXT:    v_max_f32_e32 v19, v14, v30
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX7-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX7-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_max_f32_e32 v18, v15, v16
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX7-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v16, v15, v17
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v16f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v16, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v16
+; GFX8-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX8-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v20, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v21, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v22, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v23, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v24, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v25, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v26, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v27, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v28, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v29, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v30, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX8-NEXT:    v_max_f32_e32 v1, v1, v17
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX8-NEXT:    v_max_f32_e32 v2, v2, v18
+; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX8-NEXT:    v_max_f32_e32 v18, v13, v29
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX8-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX8-NEXT:    v_max_f32_e32 v3, v3, v19
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX8-NEXT:    v_max_f32_e32 v4, v4, v20
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX8-NEXT:    v_max_f32_e32 v5, v5, v21
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX8-NEXT:    v_max_f32_e32 v6, v6, v22
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX8-NEXT:    v_max_f32_e32 v7, v7, v23
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX8-NEXT:    v_max_f32_e32 v8, v8, v24
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX8-NEXT:    v_max_f32_e32 v9, v9, v25
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX8-NEXT:    v_max_f32_e32 v10, v10, v26
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX8-NEXT:    v_max_f32_e32 v11, v11, v27
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX8-NEXT:    v_max_f32_e32 v12, v12, v28
+; GFX8-NEXT:    v_max_f32_e32 v19, v14, v30
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX8-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX8-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_max_f32_e32 v18, v15, v16
+; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v16f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v16, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v16
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX9-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v19
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v20
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v20, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v21
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v21, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v22
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v22, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v23
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v23, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v24
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v24, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v25
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v25, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v26
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v26, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v27
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v27, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v28
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v28, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v29
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v29, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v30
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v30, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v17
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX9-NEXT:    v_max_f32_e32 v2, v2, v18
+; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX9-NEXT:    v_max_f32_e32 v18, v13, v29
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX9-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX9-NEXT:    v_max_f32_e32 v3, v3, v19
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX9-NEXT:    v_max_f32_e32 v4, v4, v20
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX9-NEXT:    v_max_f32_e32 v5, v5, v21
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX9-NEXT:    v_max_f32_e32 v6, v6, v22
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX9-NEXT:    v_max_f32_e32 v7, v7, v23
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX9-NEXT:    v_max_f32_e32 v8, v8, v24
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX9-NEXT:    v_max_f32_e32 v9, v9, v25
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX9-NEXT:    v_max_f32_e32 v10, v10, v26
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX9-NEXT:    v_max_f32_e32 v11, v11, v27
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX9-NEXT:    v_max_f32_e32 v12, v12, v28
+; GFX9-NEXT:    v_max_f32_e32 v19, v14, v30
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX9-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_max_f32_e32 v18, v15, v16
+; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v16f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v0, v16
 ; GFX940-NEXT:    v_mov_b32_e32 v32, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v16, v0, vcc
+; GFX940-NEXT:    v_max_f32_e32 v33, v0, v16
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v32, v33, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v16, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v17
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX940-NEXT:    v_max_f32_e32 v34, v1, v17
+; GFX940-NEXT:    v_max_f32_e32 v35, v2, v18
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v32, v33, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
+; GFX940-NEXT:    v_max_f32_e32 v36, v3, v19
+; GFX940-NEXT:    v_max_f32_e32 v37, v4, v20
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v32, v34, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v19
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
+; GFX940-NEXT:    v_max_f32_e32 v38, v5, v21
+; GFX940-NEXT:    v_max_f32_e32 v39, v6, v22
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v32, v35, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v20
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
+; GFX940-NEXT:    v_max_f32_e32 v48, v7, v23
+; GFX940-NEXT:    v_max_f32_e32 v49, v8, v24
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v32, v36, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v20, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v21
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
+; GFX940-NEXT:    v_max_f32_e32 v50, v9, v25
+; GFX940-NEXT:    v_max_f32_e32 v51, v10, v26
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v32, v37, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v21, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v22
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
+; GFX940-NEXT:    v_max_f32_e32 v52, v11, v27
+; GFX940-NEXT:    v_max_f32_e32 v53, v12, v28
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v32, v38, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v22, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v23
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
+; GFX940-NEXT:    v_max_f32_e32 v54, v13, v29
+; GFX940-NEXT:    v_max_f32_e32 v55, v14, v30
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v32, v39, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v23, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v24
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_max_f32_e32 v16, v15, v31
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v32, v48, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v24, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v25
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v8, v32, v49, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v25, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v26
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v32, v50, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v26, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v27
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v32, v51, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v27, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v28
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v32, v52, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v28, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v29
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v12, v32, v53, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v29, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v30
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v32, v54, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v30, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v31
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v14, v32, v55, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v15, v31
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v31, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v32, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v16f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v16
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v18, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v19, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v20, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v22, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v23, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v24, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v25, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v26, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v27, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v28, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v29, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v30, v14, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v32, v0, v16
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v33, v1, v17
+; GFX10-NEXT:    v_max_f32_e32 v34, v2, v18
+; GFX10-NEXT:    v_max_f32_e32 v35, v3, v19
+; GFX10-NEXT:    v_max_f32_e32 v36, v4, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v37, v5, v21
+; GFX10-NEXT:    v_max_f32_e32 v38, v6, v22
+; GFX10-NEXT:    v_max_f32_e32 v39, v7, v23
+; GFX10-NEXT:    v_max_f32_e32 v48, v8, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v49, v9, v25
+; GFX10-NEXT:    v_max_f32_e32 v50, v10, v26
+; GFX10-NEXT:    v_max_f32_e32 v51, v11, v27
+; GFX10-NEXT:    v_max_f32_e32 v52, v12, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v53, v13, v29
+; GFX10-NEXT:    v_max_f32_e32 v54, v14, v30
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v8, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v9, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v10, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v11, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v12, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v13, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v14, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v16, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v17, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v18, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v19, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v20, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v21, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v22, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v23, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v24, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v25, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v26, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v27, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v28, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v29, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v30, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v16, v15, v31
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v15, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v31, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v16f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v16
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v34, v18, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v35, v19, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v36, v20, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v37, v21, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e32 v38, v22, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v39, v23, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v24
-; GFX11-NEXT:    v_cndmask_b32_e32 v48, v24, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v25
-; GFX11-NEXT:    v_cndmask_b32_e32 v49, v25, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v26
-; GFX11-NEXT:    v_cndmask_b32_e32 v50, v26, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v27
-; GFX11-NEXT:    v_cndmask_b32_e32 v51, v27, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v28
-; GFX11-NEXT:    v_cndmask_b32_e32 v52, v28, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v29
-; GFX11-NEXT:    v_cndmask_b32_e32 v53, v29, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v30
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, v30, v14, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v32, v0, v16 :: v_dual_max_f32 v33, v1, v17
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v34, v2, v18 :: v_dual_max_f32 v35, v3, v19
+; GFX11-NEXT:    v_dual_max_f32 v36, v4, v20 :: v_dual_max_f32 v37, v5, v21
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v54, v14, v30
+; GFX11-NEXT:    v_dual_max_f32 v38, v6, v22 :: v_dual_max_f32 v39, v7, v23
+; GFX11-NEXT:    v_dual_max_f32 v48, v8, v24 :: v_dual_max_f32 v49, v9, v25
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo
+; GFX11-NEXT:    v_dual_max_f32 v50, v10, v26 :: v_dual_max_f32 v51, v11, v27
+; GFX11-NEXT:    v_dual_max_f32 v52, v12, v28 :: v_dual_max_f32 v53, v13, v29
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v8, v24
-; GFX11-NEXT:    v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v9, v25
-; GFX11-NEXT:    v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v10, v26
-; GFX11-NEXT:    v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v11, v27
-; GFX11-NEXT:    v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v12, v28
-; GFX11-NEXT:    v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v13, v29
-; GFX11-NEXT:    v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v14, v30
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v16, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v17, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v18, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v19, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v20, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v21, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v22, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v23, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v24, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v25, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v26, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v27, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v28, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v29, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v30, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v31
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v16, v15, v31
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v15, v31
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v31, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v16f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
index d60a28e74043cc35579a4d36ad1b9b870f851be0..78fb23182f80044d8bb9e2a337851b77a07744bb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
@@ -530,221 +530,86 @@ define <2 x double> @v_maximum_v2f64(<2 x double> %src0, <2 x double> %src1) {
 ; GFX7-LABEL: v_maximum_v2f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[4:5], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v10, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v8, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[6:7]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v6, v2, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0x7ff80000, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v10, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, v12, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, v13, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s8
+; GFX10-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v6, v2, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, 0x7ff80000, v8, s1
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v10, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, 0, v13, s2
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[6:7], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[10:11]
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, 0, v12, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s3, 0, v[8:9]
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s3
+; GFX11-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[6:7]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f64:
@@ -765,182 +630,43 @@ define <2 x double> @v_maximum_v2f64__nnan(<2 x double> %src0, <2 x double> %src
 ; GFX7-LABEL: v_maximum_v2f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[4:5], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 64
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[0:1]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s8
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[6:7], 64
-; GFX11-NEXT:    v_dual_cndmask_b32 v9, v5, v1 :: v_dual_cndmask_b32 v8, v4, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s3, 0, v[8:9]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f64__nnan:
@@ -961,111 +687,86 @@ define <2 x double> @v_maximum_v2f64__nsz(<2 x double> %src0, <2 x double> %src1
 ; GFX7-LABEL: v_maximum_v2f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v8, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[6:7]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v6, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s6
+; GFX10-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[6:7]
-; GFX11-NEXT:    v_dual_cndmask_b32 v8, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
+; GFX11-NEXT:    v_max_f64 v[8:9], v[0:1], v[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[2:3], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[6:7]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v6, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f64__nsz:
@@ -1086,69 +787,43 @@ define <2 x double> @v_maximum_v2f64__nnan_nsz(<2 x double> %src0, <2 x double>
 ; GFX7-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s4
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[6:7]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f64__nnan_nsz:
@@ -1170,61 +845,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 64
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX7-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX7-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 64
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX7-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX7-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX7-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX7-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX7-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX7-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX7-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX7-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX7-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX7-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX7-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX7-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 64
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 64
-; GFX7-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX7-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX7-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX7-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX7-NEXT:    v_mov_b32_e32 v5, s9
+; GFX7-NEXT:    v_max_f64 v[2:3], s[6:7], v[0:1]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX7-NEXT:    v_max_f64 v[0:1], s[4:5], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX7-NEXT:    ;;#ASMSTART
-; GFX7-NEXT:    ; use s[4:7]
+; GFX7-NEXT:    ; use v[0:3]
 ; GFX7-NEXT:    ;;#ASMEND
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1232,61 +866,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s11
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 64
-; GFX8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX8-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX8-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX8-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 64
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX8-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX8-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX8-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX8-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX8-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX8-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX8-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX8-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX8-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX8-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX8-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 64
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 64
-; GFX8-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX8-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX8-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX8-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    v_mov_b32_e32 v5, s9
+; GFX8-NEXT:    v_max_f64 v[2:3], s[6:7], v[0:1]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX8-NEXT:    v_max_f64 v[0:1], s[4:5], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX8-NEXT:    ;;#ASMSTART
-; GFX8-NEXT:    ; use s[4:7]
+; GFX8-NEXT:    ; use v[0:3]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1294,61 +887,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
+; GFX9-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s11
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 64
-; GFX9-NEXT:    v_mov_b32_e32 v0, s8
-; GFX9-NEXT:    v_mov_b32_e32 v1, s9
-; GFX9-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX9-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX9-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 64
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX9-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX9-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX9-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX9-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX9-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX9-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX9-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX9-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX9-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 64
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 64
-; GFX9-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX9-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX9-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    v_mov_b32_e32 v5, s9
+; GFX9-NEXT:    v_max_f64 v[2:3], s[6:7], v[0:1]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX9-NEXT:    v_max_f64 v[0:1], s[4:5], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX9-NEXT:    ;;#ASMSTART
-; GFX9-NEXT:    ; use s[4:7]
+; GFX9-NEXT:    ; use v[0:3]
 ; GFX9-NEXT:    ;;#ASMEND
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1356,179 +908,52 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, s[2:3], v[0:1]
-; GFX940-NEXT:    s_and_b64 s[8:9], vcc, exec
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[8:9], s[2:3], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s12, s3, s7
-; GFX940-NEXT:    s_and_b64 s[10:11], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s11, s12, 0x7ff80000
-; GFX940-NEXT:    s_and_b64 s[12:13], vcc, exec
-; GFX940-NEXT:    s_cselect_b32 s10, s2, s6
-; GFX940-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[12:13], s[2:3], 64
-; GFX940-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX940-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[14:15], s[6:7], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[8:9], s[10:11], 0
-; GFX940-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX940-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX940-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX940-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s7, s3, s11
-; GFX940-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s11, s2, s10
-; GFX940-NEXT:    s_and_b64 s[2:3], s[14:15], exec
+; GFX940-NEXT:    v_max_f64 v[2:3], s[2:3], v[0:1]
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[2:3], v[0:1]
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
-; GFX940-NEXT:    s_cselect_b32 s6, s6, s11
-; GFX940-NEXT:    s_and_b64 s[2:3], s[8:9], exec
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s6, s6, s10
-; GFX940-NEXT:    s_and_b64 s[2:3], vcc, exec
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[2:3], s[0:1], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s10, s1, s5
-; GFX940-NEXT:    s_and_b64 s[8:9], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s9, s10, 0x7ff80000
-; GFX940-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX940-NEXT:    s_cselect_b32 s8, s0, s4
-; GFX940-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[10:11], s[0:1], 64
-; GFX940-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX940-NEXT:    s_and_b64 s[12:13], s[10:11], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[12:13], s[4:5], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[8:9], 0
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s9
-; GFX940-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX940-NEXT:    s_and_b64 s[14:15], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s5, s1, s9
-; GFX940-NEXT:    s_and_b64 s[10:11], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s9, s0, s8
-; GFX940-NEXT:    s_and_b64 s[0:1], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX940-NEXT:    s_and_b64 s[0:1], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX940-NEXT:    v_max_f64 v[4:5], s[0:1], v[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[0:1], v[0:1]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; use s[4:7]
+; GFX940-NEXT:    ; use v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: s_maximum_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s12, s[6:7], s[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s14, s[6:7], s[10:11]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s15, s[6:7], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s16, s[10:11], 64
-; GFX10-NEXT:    v_cmp_o_f64_e64 s18, s[4:5], s[8:9]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s19, s[4:5], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s20, s[8:9], 64
-; GFX10-NEXT:    s_and_b32 s13, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s13, s7, s11
-; GFX10-NEXT:    s_and_b32 s17, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s13, s13, 0x7ff80000
-; GFX10-NEXT:    s_and_b32 s12, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s12, s6, s10
-; GFX10-NEXT:    s_and_b32 s14, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s12, s12, 0
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s17, s[4:5], s[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, s[12:13], 0
-; GFX10-NEXT:    s_and_b32 s21, s15, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s13
-; GFX10-NEXT:    s_and_b32 s21, s16, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX10-NEXT:    s_and_b32 s11, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s13
-; GFX10-NEXT:    s_and_b32 s11, s15, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s12
-; GFX10-NEXT:    s_and_b32 s11, s16, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX10-NEXT:    s_and_b32 s10, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s12
-; GFX10-NEXT:    s_and_b32 s10, s17, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s5, s9
-; GFX10-NEXT:    s_and_b32 s11, s18, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s11, s10, 0x7ff80000
-; GFX10-NEXT:    s_and_b32 s10, s17, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX10-NEXT:    s_and_b32 s12, s18, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX10-NEXT:    s_and_b32 s13, s19, exec_lo
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, s[10:11], 0
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_and_b32 s13, s20, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX10-NEXT:    s_and_b32 s9, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_and_b32 s9, s19, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX10-NEXT:    s_and_b32 s9, s20, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_and_b32 s8, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX10-NEXT:    v_max_f64 v[0:1], s[6:7], s[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, s[6:7], s[10:11]
+; GFX10-NEXT:    v_max_f64 v[4:5], s[4:5], s[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, s[4:5], s[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v1, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, s4
 ; GFX10-NEXT:    ;;#ASMSTART
-; GFX10-NEXT:    ; use s[4:7]
+; GFX10-NEXT:    ; use v[0:3]
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: s_maximum_v2f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s8, s[2:3], s[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s10, s[2:3], s[6:7]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, s[2:3], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s12, s[6:7], 64
-; GFX11-NEXT:    v_cmp_o_f64_e64 s14, s[0:1], s[4:5]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s15, s[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, s[4:5], 64
-; GFX11-NEXT:    s_and_b32 s9, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s9, s3, s7
-; GFX11-NEXT:    s_and_b32 s13, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s9, s9, 0x7ff80000
-; GFX11-NEXT:    s_and_b32 s8, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s8, s2, s6
-; GFX11-NEXT:    s_and_b32 s10, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s13, s[0:1], s[4:5]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, s[8:9], 0
-; GFX11-NEXT:    s_and_b32 s17, s11, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX11-NEXT:    s_and_b32 s17, s12, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s7, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX11-NEXT:    s_and_b32 s7, s11, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX11-NEXT:    s_and_b32 s7, s12, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX11-NEXT:    s_and_b32 s6, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    s_and_b32 s6, s13, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s1, s5
-; GFX11-NEXT:    s_and_b32 s7, s14, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s7, s6, 0x7ff80000
-; GFX11-NEXT:    s_and_b32 s6, s13, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s0, s4
-; GFX11-NEXT:    s_and_b32 s8, s14, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s6, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    s_and_b32 s9, s15, exec_lo
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, s[6:7], 0
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX11-NEXT:    s_and_b32 s9, s16, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX11-NEXT:    s_and_b32 s5, s15, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s6
-; GFX11-NEXT:    s_and_b32 s5, s16, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s4, s0
-; GFX11-NEXT:    s_and_b32 s4, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s6
+; GFX11-NEXT:    v_max_f64 v[0:1], s[2:3], s[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[6:7]
+; GFX11-NEXT:    v_max_f64 v[4:5], s[0:1], s[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[4:5]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v1, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, s0
 ; GFX11-NEXT:    ;;#ASMSTART
-; GFX11-NEXT:    ; use s[0:3]
+; GFX11-NEXT:    ; use v[0:3]
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1554,306 +979,110 @@ define <3 x double> @v_maximum_v3f64(<3 x double> %src0, <3 x double> %src1) {
 ; GFX7-LABEL: v_maximum_v3f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 64
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 64
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 64
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v11, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX940-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v10, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v12, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v12, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[8:9]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v12, vcc
+; GFX940-NEXT:    v_max_f64 v[6:7], v[4:5], v[10:11]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v12, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v10, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0x7ff80000, v12, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, 0x7ff80000, v14, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, v17, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, v18, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[8:9], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[6:7], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[12:13]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[14:15]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[16:17]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s11
+; GFX10-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX10-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v12, v7, v1 :: v_dual_cndmask_b32 v17, v6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v10, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, 0x7ff80000, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, 0x7ff80000, v14, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, 0, v17, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, 0, v18, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s4
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[8:9], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[10:11], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[12:13]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[14:15]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s7
+; GFX11-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX11-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[10:11]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f64:
@@ -1875,247 +1104,49 @@ define <3 x double> @v_maximum_v3f64__nnan(<3 x double> %src0, <3 x double> %src
 ; GFX7-LABEL: v_maximum_v3f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 64
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[12:13]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 64
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[0:1]
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v11, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v10, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[8:9], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[6:7], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v10, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[12:13]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[14:15]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[16:17]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s11
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[8:9], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[10:11], 64
-; GFX11-NEXT:    v_dual_cndmask_b32 v13, v7, v1 :: v_dual_cndmask_b32 v12, v6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v10, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[12:13]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[14:15]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s7
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f64__nnan:
@@ -2137,144 +1168,110 @@ define <3 x double> @v_maximum_v3f64__nsz(<3 x double> %src0, <3 x double> %src1
 ; GFX7-LABEL: v_maximum_v3f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX7-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX8-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX9-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v6, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v6, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v10, v4, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; GFX940-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX940-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v6, v5, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v12, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v12, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[8:9]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v12, vcc
+; GFX940-NEXT:    v_max_f64 v[6:7], v[4:5], v[10:11]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v12, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v10, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s8
+; GFX10-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX10-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v12, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v10, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s4
+; GFX11-NEXT:    v_max_f64 v[12:13], v[0:1], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX11-NEXT:    v_max_f64 v[6:7], v[2:3], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[8:9], v[4:5], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[10:11]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f64__nsz:
@@ -2296,88 +1293,49 @@ define <3 x double> @v_maximum_v3f64__nnan_nsz(<3 x double> %src0, <3 x double>
 ; GFX7-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s5
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s1
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[4:5], v[10:11]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f64__nnan_nsz:
@@ -2399,404 +1357,135 @@ define <4 x double> @v_maximum_v4f64(<4 x double> %src0, <4 x double> %src1) {
 ; GFX7-LABEL: v_maximum_v4f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX7-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX7-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX7-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX8-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX8-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX8-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX9-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX9-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[12:13], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[14:15], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v16, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v16, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v16, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[4:5], v[12:13]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[12:13]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v16, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[6:7], v[14:15]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[14:15]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s6, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s8, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s10, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[14:15], 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v13, v5, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v15, v7, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, 0x7ff80000, v18, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v12, v4, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, v21, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v14, v6, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[4:5], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[6:7], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0x7ff80000, v20, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, v23, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0x7ff80000, v22, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, v24, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s9, v[10:11], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[16:17]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[18:19]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[20:21]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[22:23]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s14
+; GFX10-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[10:11]
+; GFX10-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[12:13]
+; GFX10-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s2, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s3, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s6, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v11, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v13, v5, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v15, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, 0x7ff80000, v18, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, 0x7ff80000, v20, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v14, v6, s3
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, 0x7ff80000, v22, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, 0, v20, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, 0, v24, s6
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[12:13], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[14:15], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[10:11], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[18:19]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[20:21]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[22:23]
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s8
+; GFX11-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[10:11]
+; GFX11-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[12:13]
+; GFX11-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[14:15]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f64:
@@ -2819,320 +1508,55 @@ define <4 x double> @v_maximum_v4f64__nnan(<4 x double> %src0, <4 x double> %src
 ; GFX7-LABEL: v_maximum_v4f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX7-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX8-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX9-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 64
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v13, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[12:13], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[0:1]
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v15, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[14:15], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX940-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s6, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[10:11], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[8:9], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[14:15], 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v15, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[4:5], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v14, v6, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[6:7], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[2:3], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[16:17]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[18:19]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[20:21]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[22:23]
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v20, v4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s14
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX10-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s2, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[14:15], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[10:11], 64
-; GFX11-NEXT:    v_dual_cndmask_b32 v17, v9, v1 :: v_dual_cndmask_b32 v16, v8, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v13, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, v15, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v14, v6, s2
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[12:13], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[18:19]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[20:21]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[22:23]
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s4
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v16, v0 :: v_dual_cndmask_b32 v1, v17, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s10
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f64__nnan:
@@ -3155,180 +1579,135 @@ define <4 x double> @v_maximum_v4f64__nsz(<4 x double> %src0, <4 x double> %src1
 ; GFX7-LABEL: v_maximum_v4f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX7-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX7-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX8-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX8-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX9-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX9-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v12, v4, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v15, v7, vcc
+; GFX940-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v16, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v16, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v16, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[4:5], v[12:13]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[12:13]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v16, vcc
+; GFX940-NEXT:    v_max_f64 v[8:9], v[6:7], v[14:15]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[14:15]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s7, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s10, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v12, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v14, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, v10, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0x7ff80000, v7, s10
+; GFX10-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[10:11]
+; GFX10-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[12:13]
+; GFX10-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s2, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s3, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s6, v[6:7], v[14:15]
-; GFX11-NEXT:    v_dual_cndmask_b32 v16, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v12, v4, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v14, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, v10, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, 0, v12, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, 0x7ff80000, v7, s6
+; GFX11-NEXT:    v_max_f64 v[16:17], v[0:1], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[8:9], v[2:3], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[10:11]
+; GFX11-NEXT:    v_max_f64 v[10:11], v[4:5], v[12:13]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[12:13]
+; GFX11-NEXT:    v_max_f64 v[12:13], v[6:7], v[14:15]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[14:15]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f64__nsz:
@@ -3351,108 +1730,55 @@ define <4 x double> @v_maximum_v4f64__nnan_nsz(<4 x double> %src0, <4 x double>
 ; GFX7-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX7-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX8-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX9-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v15, v7, vcc
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX940-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX940-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX940-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s6, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s6
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX10-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX10-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX10-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s2, v[6:7], v[14:15]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s2
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-NEXT:    v_max_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-NEXT:    v_max_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-NEXT:    v_max_f64 v[6:7], v[6:7], v[14:15]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f64__nnan_nsz:
@@ -3475,782 +1801,244 @@ define <8 x double> @v_maximum_v8f64(<8 x double> %src0, <8 x double> %src1) {
 ; GFX7-LABEL: v_maximum_v8f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 64
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX7-NEXT:    v_max_f64 v[32:33], v[2:3], v[18:19]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX7-NEXT:    v_max_f64 v[18:19], v[4:5], v[20:21]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[0:1], v[16:17]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX7-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX7-NEXT:    v_max_f64 v[20:21], v[6:7], v[22:23]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX7-NEXT:    v_max_f64 v[16:17], v[8:9], v[24:25]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX7-NEXT:    v_max_f64 v[22:23], v[10:11], v[26:27]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX7-NEXT:    v_max_f64 v[24:25], v[12:13], v[28:29]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX7-NEXT:    v_max_f64 v[18:19], v[14:15], v[30:31]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v8f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 64
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX8-NEXT:    v_max_f64 v[32:33], v[2:3], v[18:19]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX8-NEXT:    v_max_f64 v[18:19], v[4:5], v[20:21]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[0:1], v[16:17]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX8-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX8-NEXT:    v_max_f64 v[20:21], v[6:7], v[22:23]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX8-NEXT:    v_max_f64 v[16:17], v[8:9], v[24:25]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX8-NEXT:    v_max_f64 v[22:23], v[10:11], v[26:27]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX8-NEXT:    v_max_f64 v[24:25], v[12:13], v[28:29]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX8-NEXT:    v_max_f64 v[18:19], v[14:15], v[30:31]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v8f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 64
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX9-NEXT:    v_max_f64 v[32:33], v[2:3], v[18:19]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX9-NEXT:    v_max_f64 v[18:19], v[4:5], v[20:21]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[0:1], v[16:17]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX9-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX9-NEXT:    v_max_f64 v[20:21], v[6:7], v[22:23]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX9-NEXT:    v_max_f64 v[16:17], v[8:9], v[24:25]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX9-NEXT:    v_max_f64 v[22:23], v[10:11], v[26:27]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX9-NEXT:    v_max_f64 v[24:25], v[12:13], v[28:29]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX9-NEXT:    v_max_f64 v[18:19], v[14:15], v[30:31]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v8f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX940-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v35, v32, v33, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v34, 0, v33, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[16:17], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v34, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v35, v1, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[18:19]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v34, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v35, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[18:19], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[20:21]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v16, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[20:21]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[20:21], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[22:23]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[22:23], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[8:9], v[24:25]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v16, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[8:9], v[24:25]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[24:25], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[10:11], v[26:27]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[26:27], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[12:13], v[28:29]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v16, v10, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[12:13], v[28:29]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[28:29], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v54, 0x7ff80000
+; GFX940-NEXT:    v_max_f64 v[32:33], v[0:1], v[16:17]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[16:17]
+; GFX940-NEXT:    v_max_f64 v[34:35], v[2:3], v[18:19]
+; GFX940-NEXT:    v_max_f64 v[36:37], v[4:5], v[20:21]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v33, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX940-NEXT:    v_max_f64 v[38:39], v[6:7], v[22:23]
+; GFX940-NEXT:    v_max_f64 v[48:49], v[8:9], v[24:25]
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v34, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v35, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[20:21]
+; GFX940-NEXT:    v_max_f64 v[50:51], v[10:11], v[26:27]
+; GFX940-NEXT:    v_max_f64 v[52:53], v[12:13], v[28:29]
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v36, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v37, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[22:23]
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[14:15], v[30:31]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[30:31], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v15, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v16, v14, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v17, v15, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[16:17], v[14:15], v[30:31]
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v38, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v39, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[8:9], v[24:25]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v8, v48, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v49, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[10:11], v[26:27]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v10, v50, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v51, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[12:13], v[28:29]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v12, v52, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v53, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v14, v16, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v54, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v8f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[16:17]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[0:1], v[16:17]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s9, v[6:7], v[22:23]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s10, v[8:9], v[24:25]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s11, v[10:11], v[26:27]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s12, v[12:13], v[28:29]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s13, v[6:7], v[22:23]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s14, v[8:9], v[24:25]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s15, v[10:11], v[26:27]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s16, v[12:13], v[28:29]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[2:3], v[18:19]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[18:19]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s7, v[4:5], v[20:21]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[20:21]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[26:27], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s18, v[28:29], 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v23, v7, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v25, v9, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v27, v11, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v33, 0x7ff80000, v32, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, v29, v13, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v38, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v48, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, 0, v32, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v50, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v52, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v22, v6, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v24, v8, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v26, v10, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, v28, v12, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[16:17], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[18:19], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v19, v3, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, v21, v5, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 s9, v[12:13], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v35, 0x7ff80000, v34, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v18, v2, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, v20, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[8:9], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s16
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[20:21], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s15, v[22:23], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s16, v[24:25], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[32:33]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[34:35]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[36:37]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[48:49]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[50:51]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[52:53]
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s24
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s24
+; GFX10-NEXT:    v_max_f64 v[32:33], v[0:1], v[16:17]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17]
+; GFX10-NEXT:    v_max_f64 v[16:17], v[2:3], v[18:19]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[18:19]
+; GFX10-NEXT:    v_max_f64 v[18:19], v[4:5], v[20:21]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[20:21]
+; GFX10-NEXT:    v_max_f64 v[20:21], v[6:7], v[22:23]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[22:23]
+; GFX10-NEXT:    v_max_f64 v[22:23], v[8:9], v[24:25]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s7, v[8:9], v[24:25]
+; GFX10-NEXT:    v_max_f64 v[24:25], v[10:11], v[26:27]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s8, v[10:11], v[26:27]
+; GFX10-NEXT:    v_max_f64 v[26:27], v[12:13], v[28:29]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s9, v[12:13], v[28:29]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v16, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v17, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v19, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v21, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v22, 0, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v23, 0x7ff80000, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v24, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v25, 0x7ff80000, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v26, 0, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v27, 0x7ff80000, s9
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s10, v[14:15], v[30:31]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s13, v[14:15], v[30:31]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s25, v[30:31], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v31, v15, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v30, v14, s10
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v16, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, 0, v18, s13
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[14:15], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e32 vcc_lo, 0, v[54:55]
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s25
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s25
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v55, v15, vcc_lo
+; GFX10-NEXT:    v_max_f64 v[28:29], v[14:15], v[30:31]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s10, v[14:15], v[30:31]
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v28, 0, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v29, 0x7ff80000, s10
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v8f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s4, v[6:7], v[22:23]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s9, v[6:7], v[22:23]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s1, v[2:3], v[18:19]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s6, v[10:11], v[26:27]
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[16:17]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[18:19]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s11, v[10:11], v[26:27]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[16:17]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s3, v[4:5], v[20:21]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s5, v[8:9], v[24:25]
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s7, v[12:13], v[28:29]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[20:21]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s10, v[8:9], v[24:25]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s12, v[12:13], v[28:29]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s13, v[18:19], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s15, v[20:21], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, v23, v7, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, v19, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, v27, v11, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v38, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, v22, v6, s4
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[6:7], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v35, 0x7ff80000, v34, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v50, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, v18, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, v26, v10, s6
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[0:1], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, v21, v5, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, v25, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, v29, v13, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s11
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, v[16:17], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v48, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v52, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, v20, v4, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, v24, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, v28, v12, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s9
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[2:3], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[4:5], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s7, v[10:11], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s9, v[12:13], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s12
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[24:25], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, v[26:27], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[28:29], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[34:35]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s16, 0, v[36:37]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[38:39]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s18, 0, v[48:49]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[50:51]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[52:53]
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s4
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v33, 0x7ff80000, v32, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v32, 0, v32, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s1
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[32:33]
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s12
+; GFX11-NEXT:    v_max_f64 v[32:33], v[0:1], v[16:17]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17]
+; GFX11-NEXT:    v_max_f64 v[16:17], v[2:3], v[18:19]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[18:19]
+; GFX11-NEXT:    v_max_f64 v[18:19], v[4:5], v[20:21]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[20:21]
+; GFX11-NEXT:    v_max_f64 v[20:21], v[6:7], v[22:23]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[22:23]
+; GFX11-NEXT:    v_max_f64 v[22:23], v[8:9], v[24:25]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s3, v[8:9], v[24:25]
+; GFX11-NEXT:    v_max_f64 v[24:25], v[10:11], v[26:27]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s4, v[10:11], v[26:27]
+; GFX11-NEXT:    v_max_f64 v[26:27], v[12:13], v[28:29]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s5, v[12:13], v[28:29]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v16, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v17, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v19, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v21, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v8, v22, 0, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v23, 0x7ff80000, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v24, 0, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v25, 0x7ff80000, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v26, 0, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v27, 0x7ff80000, s5
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[14:15], v[30:31]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[14:15], v[30:31]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s19, v[30:31], 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, v31, v15, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v54, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v54, 0, v16, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[14:15], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[54:55]
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_cndmask_b32 v6, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s17
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s19
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s22
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s22
+; GFX11-NEXT:    v_max_f64 v[28:29], v[14:15], v[30:31]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s6, v[14:15], v[30:31]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v28, 0, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v29, 0x7ff80000, s6
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v8f64:
@@ -4279,1799 +2067,798 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-LABEL: v_maximum_v16f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX7-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX7-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX7-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX7-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX7-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX7-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX7-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX7-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX7-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX7-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[2:3], v[2:3], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[4:5], v[4:5], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[6:7], v[6:7], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[8:9], v[8:9], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[10:11], v[10:11], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 64
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX7-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX7-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[12:13], v[12:13], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[14:15], v[14:15], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[16:17], v[16:17], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX7-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[18:19], v[18:19], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 64
-; GFX7-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX7-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX7-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX7-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX7-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[20:21], v[20:21], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX7-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[22:23], v[22:23], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX7-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[24:25], v[24:25], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX7-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[26:27], v[26:27], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX7-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 64
-; GFX7-NEXT:    s_waitcnt vmcnt(7)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(5)
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX7-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX7-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX7-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
+; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX7-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX7-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX7-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX7-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX7-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX7-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX7-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX7-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 64
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v16f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX8-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX8-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX8-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX8-NEXT:    s_waitcnt vmcnt(2)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[2:3], v[2:3], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[4:5], v[4:5], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX8-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX8-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[6:7], v[6:7], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[8:9], v[8:9], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[10:11], v[10:11], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 64
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX8-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX8-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[12:13], v[12:13], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[14:15], v[14:15], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[16:17], v[16:17], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[18:19], v[18:19], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 64
-; GFX8-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX8-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX8-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX8-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX8-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[20:21], v[20:21], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX8-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[22:23], v[22:23], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[24:25], v[24:25], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX8-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[26:27], v[26:27], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX8-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 64
-; GFX8-NEXT:    s_waitcnt vmcnt(7)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(5)
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX8-NEXT:    s_waitcnt vmcnt(3)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX8-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX8-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
+; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX8-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX8-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX8-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX8-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX8-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX8-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX8-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 64
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v16f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX9-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX9-NEXT:    v_writelane_b32 v34, s35, 3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX9-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[6:7], v[6:7], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[8:9], v[8:9], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[10:11], v[10:11], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 64
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[12:13], v[12:13], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[14:15], v[14:15], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[16:17], v[16:17], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[18:19], v[18:19], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 64
-; GFX9-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[20:21], v[20:21], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[22:23], v[22:23], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[24:25], v[24:25], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[26:27], v[26:27], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX9-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX9-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 64
-; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX9-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 64
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX9-NEXT:    v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX9-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX9-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
+; GFX9-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX9-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX9-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 64
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v16f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:8
-; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:4
-; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:16
-; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:12
-; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:24
-; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:20
-; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:32
-; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:28
+; GFX940-NEXT:    v_accvgpr_write_b32 a1, v40 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a2, v41 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a3, v42 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a4, v43 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a5, v44 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a6, v45 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a7, v46 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a8, v47 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a9, v56 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a10, v57 ; Reload Reuse
+; GFX940-NEXT:    scratch_load_dword v37, off, s32 offset:16
+; GFX940-NEXT:    scratch_load_dword v36, off, s32 offset:12
+; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:24
+; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:20
+; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:32
+; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:28
+; GFX940-NEXT:    scratch_load_dword v57, off, s32 offset:8
+; GFX940-NEXT:    scratch_load_dword v56, off, s32 offset:4
+; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:40
+; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:36
+; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:48
+; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:44
+; GFX940-NEXT:    scratch_load_dword v43, off, s32 offset:56
+; GFX940-NEXT:    scratch_load_dword v42, off, s32 offset:52
+; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:64
+; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:60
+; GFX940-NEXT:    scratch_load_dword v55, off, s32 offset:72
+; GFX940-NEXT:    scratch_load_dword v54, off, s32 offset:68
+; GFX940-NEXT:    scratch_load_dword v53, off, s32 offset:80
+; GFX940-NEXT:    scratch_load_dword v52, off, s32 offset:76
+; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:88
+; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:84
+; GFX940-NEXT:    scratch_load_dword v35, off, s32 offset:96
+; GFX940-NEXT:    scratch_load_dword v34, off, s32 offset:92
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    scratch_load_dword v33, off, s32 offset:128
-; GFX940-NEXT:    scratch_load_dword v32, off, s32 offset:124
-; GFX940-NEXT:    scratch_load_dword v35, off, s32 offset:120
-; GFX940-NEXT:    scratch_load_dword v34, off, s32 offset:116
-; GFX940-NEXT:    scratch_load_dword v43, off, s32 offset:40
-; GFX940-NEXT:    scratch_load_dword v42, off, s32 offset:36
+; GFX940-NEXT:    scratch_load_dword v33, off, s32 offset:104
+; GFX940-NEXT:    scratch_load_dword v32, off, s32 offset:100
+; GFX940-NEXT:    v_accvgpr_write_b32 a11, v58 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a12, v59 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a13, v60 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a14, v61 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a15, v62 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a16, v63 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_max_f64 v[58:59], v[2:3], v[36:37]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[36:37]
 ; GFX940-NEXT:    scratch_load_dword v37, off, s32 offset:112
 ; GFX940-NEXT:    scratch_load_dword v36, off, s32 offset:108
-; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:104
-; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:100
-; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:96
-; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:92
-; GFX940-NEXT:    scratch_load_dword v53, off, s32 offset:56
-; GFX940-NEXT:    scratch_load_dword v52, off, s32 offset:52
-; GFX940-NEXT:    scratch_load_dword v55, off, s32 offset:48
-; GFX940-NEXT:    scratch_load_dword v54, off, s32 offset:44
-; GFX940-NEXT:    v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX940-NEXT:    v_mov_b32_e32 v56, 0x7ff80000
-; GFX940-NEXT:    v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a13, v61 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_max_f64 v[60:61], v[4:5], v[38:39]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[0:1], v[4:5], v[38:39]
+; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:120
+; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:116
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_max_f64 v[62:63], v[6:7], v[48:49]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[2:3], v[6:7], v[48:49]
+; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:128
+; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:124
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_max_f64 v[2:3], v[0:1], v[56:57]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[4:5], v[0:1], v[56:57]
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x7ff80000
 ; GFX940-NEXT:    s_waitcnt vmcnt(23)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[0:1], v[40:41]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v41, v1, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[40:41]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v40, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v57, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[40:41], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v58, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v59, v1, vcc
+; GFX940-NEXT:    v_max_f64 v[56:57], v[8:9], v[46:47]
+; GFX940-NEXT:    v_cndmask_b32_e64 v1, v2, 0, s[4:5]
+; GFX940-NEXT:    v_accvgpr_write_b32 a0, v1
+; GFX940-NEXT:    v_cndmask_b32_e64 v1, v3, v0, s[4:5]
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v58, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v59, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[8:9], v[46:47]
 ; GFX940-NEXT:    s_waitcnt vmcnt(21)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[2:3], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v41, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v51, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e32 v40, v50, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 64
-; GFX940-NEXT:    v_cndmask_b32_e64 v61, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v60, 0, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v60, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v61, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[50:51], 64
-; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:64
-; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:60
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v50, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v51, vcc
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(21)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[44:45]
-; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:88
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v58, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v59, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[60:61]
+; GFX940-NEXT:    v_max_f64 v[46:47], v[10:11], v[44:45]
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v60, 0, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v8, v56, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v57, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[10:11], v[44:45]
+; GFX940-NEXT:    s_waitcnt vmcnt(19)
+; GFX940-NEXT:    v_max_f64 v[44:45], v[12:13], v[42:43]
+; GFX940-NEXT:    v_cndmask_b32_e64 v5, v61, v0, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v10, v46, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v47, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[12:13], v[42:43]
+; GFX940-NEXT:    s_waitcnt vmcnt(17)
+; GFX940-NEXT:    v_max_f64 v[42:43], v[14:15], v[40:41]
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v62, 0, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v12, v44, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v45, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[40:41]
+; GFX940-NEXT:    s_waitcnt vmcnt(15)
+; GFX940-NEXT:    v_max_f64 v[40:41], v[16:17], v[54:55]
+; GFX940-NEXT:    v_cndmask_b32_e64 v7, v63, v0, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v14, v42, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v43, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[16:17], v[54:55]
+; GFX940-NEXT:    s_waitcnt vmcnt(13)
+; GFX940-NEXT:    v_max_f64 v[54:55], v[18:19], v[52:53]
+; GFX940-NEXT:    v_accvgpr_read_b32 v63, a16 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v16, v40, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v17, v41, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[18:19], v[52:53]
+; GFX940-NEXT:    s_waitcnt vmcnt(11)
+; GFX940-NEXT:    v_max_f64 v[52:53], v[20:21], v[50:51]
+; GFX940-NEXT:    v_accvgpr_read_b32 v62, a15 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v18, v54, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v19, v55, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[20:21], v[50:51]
+; GFX940-NEXT:    s_waitcnt vmcnt(9)
+; GFX940-NEXT:    v_max_f64 v[50:51], v[22:23], v[34:35]
+; GFX940-NEXT:    v_accvgpr_read_b32 v61, a14 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v20, v52, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v21, v53, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[22:23], v[34:35]
+; GFX940-NEXT:    s_waitcnt vmcnt(6)
+; GFX940-NEXT:    v_max_f64 v[34:35], v[24:25], v[32:33]
+; GFX940-NEXT:    v_accvgpr_read_b32 v60, a13 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v22, v50, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v23, v51, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[24:25], v[32:33]
+; GFX940-NEXT:    v_accvgpr_read_b32 v59, a12 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v58, a11 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v24, v34, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v25, v35, v0, vcc
+; GFX940-NEXT:    v_accvgpr_read_b32 v57, a10 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v56, a9 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v47, a8 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v46, a7 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v45, a6 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v44, a5 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v43, a4 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v42, a3 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v41, a2 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v40, a1 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(4)
+; GFX940-NEXT:    v_max_f64 v[32:33], v[26:27], v[36:37]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[26:27], v[36:37]
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v60, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v61, v3, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[4:5], v[44:45]
-; GFX940-NEXT:    v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v60, a12 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v45, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v44, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[44:45], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v58, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v59, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v44, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v45, s[0:1]
-; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:72
-; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:68
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(22)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v58, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v59, v5, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[6:7], v[46:47]
+; GFX940-NEXT:    v_cndmask_b32_e64 v26, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v27, v33, v0, vcc
+; GFX940-NEXT:    s_waitcnt vmcnt(2)
+; GFX940-NEXT:    v_max_f64 v[32:33], v[28:29], v[38:39]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[28:29], v[38:39]
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v47, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v46, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[46:47], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v58, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v59, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v46, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v47, s[0:1]
-; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:80
-; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:76
-; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:84
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(18)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[8:9], v[42:43]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v58, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v59, v7, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[8:9], v[42:43]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v43, v9, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v42, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v57, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[42:43], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[58:59]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v58, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v59, v9, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(8)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[10:11], v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v8, v42, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v9, v43, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v42, v55, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[10:11], v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v58, v8, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v59, v9, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v43, v56, v42, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v42, v54, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v42, 0, v42, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[54:55], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[42:43]
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v42, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v43, v11, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[12:13], v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v10, v54, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v11, v55, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v54, v53, v13, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[12:13], v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v42, v10, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v43, v11, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v55, v56, v54, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v54, v52, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v54, 0, v54, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[52:53], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(6)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[14:15], v[40:41]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v12, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v13, v53, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v41, v15, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[14:15], v[40:41]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v54, v12, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v55, v13, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v40, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[40:41], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v52, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v14, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v53, v15, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(3)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[16:17], v[44:45]
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v15, v41, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v45, v17, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[16:17], v[44:45]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v44, v16, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[44:45], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v52, v16, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, v16, v44, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v17, v53, v17, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(1)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[18:19], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, v52, v16, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v17, v45, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v47, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[18:19], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v53, v17, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v46, v18, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[46:47], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v52, v18, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v18, v18, v46, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v53, v19, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v28, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v29, v33, v0, vcc
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[20:21], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v18, v52, v18, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v19, v19, v47, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v51, v21, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[20:21], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v19, v53, v19, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v50, v20, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[20:21], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[50:51], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v20, v52, v20, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v21, v53, v21, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[22:23], v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e64 v20, v20, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v21, v21, v51, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v49, v23, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[22:23], v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e64 v20, v52, v20, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v21, v53, v21, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v51, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v48, v22, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[48:49], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e32 v22, v50, v22, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v23, v51, v23, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[24:25], v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e64 v22, v22, v48, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v23, v23, v49, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v48, v39, v25, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[24:25], v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e64 v22, v50, v22, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v23, v51, v23, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v49, v56, v48, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v48, v38, v24, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[38:39], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e32 v24, v48, v24, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v25, v49, v25, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[26:27], v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e64 v24, v24, v38, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v25, v25, v39, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v38, v37, v27, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[26:27], v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e64 v24, v48, v24, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v25, v49, v25, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v39, v56, v38, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v38, v36, v26, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[36:37], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e32 v26, v38, v26, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v27, v39, v27, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[28:29], v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e64 v26, v26, v36, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v27, v27, v37, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v36, v35, v29, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[28:29], v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e64 v26, v38, v26, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v27, v39, v27, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v37, v56, v36, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v36, v34, v28, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[28:29], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[34:35], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e32 v28, v36, v28, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v29, v37, v29, vcc
-; GFX940-NEXT:    v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX940-NEXT:    v_cndmask_b32_e64 v28, v28, v34, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v29, v29, v35, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v34, v33, v31, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[30:31], v[32:33]
-; GFX940-NEXT:    v_cndmask_b32_e64 v28, v36, v28, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v29, v37, v29, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v35, v56, v34, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v34, v32, v30, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[32:33], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e32 v30, v34, v30, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v31, v35, v31, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v30, v34, v30, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v31, v35, v31, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX940-NEXT:    v_max_f64 v[32:33], v[30:31], v[48:49]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[30:31], v[48:49]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v30, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v31, v33, v0, vcc
+; GFX940-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v16f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x20
+; GFX10-NEXT:    s_clause 0x19
+; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:32
+; GFX10-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:28
+; GFX10-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
+; GFX10-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:68
+; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; GFX10-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
+; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:56
+; GFX10-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:52
+; GFX10-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:48
+; GFX10-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:44
+; GFX10-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
 ; GFX10-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:8
 ; GFX10-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:4
-; GFX10-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:16
-; GFX10-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:12
-; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:24
-; GFX10-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:20
-; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:32
-; GFX10-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:28
-; GFX10-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:36
-; GFX10-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:40
-; GFX10-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:56
-; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:52
-; GFX10-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
-; GFX10-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:44
-; GFX10-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:60
-; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:72
-; GFX10-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:80
-; GFX10-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:76
-; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:88
-; GFX10-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:84
+; GFX10-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:100
 ; GFX10-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:96
 ; GFX10-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:92
-; GFX10-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:100
-; GFX10-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:104
-; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:112
-; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:108
-; GFX10-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:120
-; GFX10-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:116
+; GFX10-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:88
+; GFX10-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:80
+; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:76
+; GFX10-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:72
+; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    v_max_f64 v[82:83], v[2:3], v[31:32]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[2:3], v[31:32]
+; GFX10-NEXT:    s_waitcnt vmcnt(22)
+; GFX10-NEXT:    v_max_f64 v[84:85], v[4:5], v[33:34]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[4:5], v[33:34]
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:120
+; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:116
+; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:112
+; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:108
+; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    v_max_f64 v[32:33], v[6:7], v[35:36]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[6:7], v[35:36]
+; GFX10-NEXT:    s_clause 0x2
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:128
-; GFX10-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:124
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[2:3], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[10:11], 64
-; GFX10-NEXT:    s_waitcnt vmcnt(31)
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[64:65]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[0:1], v[64:65]
-; GFX10-NEXT:    s_waitcnt vmcnt(29)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s5, v[2:3], v[54:55]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[54:55]
-; GFX10-NEXT:    s_waitcnt vmcnt(27)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s7, v[4:5], v[52:53]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[52:53]
-; GFX10-NEXT:    s_waitcnt vmcnt(25)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s9, v[6:7], v[50:51]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[6:7], v[50:51]
+; GFX10-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:128
+; GFX10-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:124
 ; GFX10-NEXT:    s_waitcnt vmcnt(23)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s13, v[8:9], v[48:49]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[64:65], 64
+; GFX10-NEXT:    v_cmp_u_f64_e64 s10, v[14:15], v[50:51]
 ; GFX10-NEXT:    s_waitcnt vmcnt(21)
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s15, v[12:13], v[36:37]
-; GFX10-NEXT:    s_waitcnt vmcnt(17)
-; GFX10-NEXT:    v_cmp_o_f64_e64 s16, v[14:15], v[34:35]
-; GFX10-NEXT:    v_cndmask_b32_e32 v96, v64, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v97, v54, v2, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v99, v55, v3, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v100, v52, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v96, 0, v96, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v101, v50, v6, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v98, 0, v97, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v97, v65, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[54:55], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v99, 0x7ff80000, v99, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v97, 0x7ff80000, v97, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v100, 0, v100, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v102, 0, v101, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[6:7], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[4:5], 64
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[10:11], v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v48, v8, s13
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[12:13], v[36:37]
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s6, v[14:15], v[34:35]
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v64, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v65, s14
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[52:53], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v113, v36, v12, s15
+; GFX10-NEXT:    v_cmp_u_f64_e64 s9, v[12:13], v[52:53]
+; GFX10-NEXT:    s_waitcnt vmcnt(19)
+; GFX10-NEXT:    v_cmp_u_f64_e64 s7, v[10:11], v[54:55]
+; GFX10-NEXT:    s_waitcnt vmcnt(18)
+; GFX10-NEXT:    v_max_f64 v[34:35], v[8:9], v[37:38]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[8:9], v[37:38]
+; GFX10-NEXT:    s_waitcnt vmcnt(16)
+; GFX10-NEXT:    v_max_f64 v[8:9], v[0:1], v[64:65]
+; GFX10-NEXT:    v_max_f64 v[36:37], v[10:11], v[54:55]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s8, v[0:1], v[64:65]
+; GFX10-NEXT:    v_max_f64 v[38:39], v[12:13], v[52:53]
+; GFX10-NEXT:    v_max_f64 v[52:53], v[14:15], v[50:51]
+; GFX10-NEXT:    s_waitcnt vmcnt(11)
+; GFX10-NEXT:    v_max_f64 v[54:55], v[20:21], v[70:71]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s13, v[20:21], v[70:71]
+; GFX10-NEXT:    s_waitcnt vmcnt(9)
+; GFX10-NEXT:    v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
+; GFX10-NEXT:    s_waitcnt vmcnt(8)
+; GFX10-NEXT:    v_max_f64 v[50:51], v[16:17], v[48:49]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s11, v[16:17], v[48:49]
+; GFX10-NEXT:    v_max_f64 v[48:49], v[18:19], v[80:81]
+; GFX10-NEXT:    v_max_f64 v[64:65], v[22:23], v[68:69]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
+; GFX10-NEXT:    s_waitcnt vmcnt(7)
+; GFX10-NEXT:    v_max_f64 v[68:69], v[24:25], v[66:67]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v36, 0, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v34, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v35, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v37, 0x7ff80000, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v38, 0, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v39, 0x7ff80000, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, 0, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, 0x7ff80000, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, v50, 0, s11
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, v51, 0x7ff80000, s11
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, v48, 0, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v49, 0x7ff80000, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v20, v54, 0, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v21, v55, 0x7ff80000, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v22, v64, 0, s14
+; GFX10-NEXT:    v_cndmask_b32_e64 v23, v65, 0x7ff80000, s14
+; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
+; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
+; GFX10-NEXT:    s_waitcnt vmcnt(5)
+; GFX10-NEXT:    v_max_f64 v[70:71], v[28:29], v[2:3]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(3)
+; GFX10-NEXT:    v_max_f64 v[66:67], v[26:27], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v82, 0, vcc_lo
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_o_f64_e64 s18, v[30:31], v[86:87]
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v54, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, v53, v5, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[50:51], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, v51, v7, s9
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[8:9], v[48:49]
-; GFX10-NEXT:    v_cndmask_b32_e64 v101, 0x7ff80000, v54, s8
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s7, v[16:17], v[32:33]
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v103, 0x7ff80000, v55, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[8:9], 64
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[10:11], v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[48:49], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v38, v10, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, v34, v14, s6
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[16:17], v[32:33]
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v52, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v53, s14
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s14, v[18:19], v[82:83]
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, 0, v115, s16
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v50, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v49, v9, s13
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[38:39], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, 0, v112, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, v39, v11, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v50, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, 0, v113, s5
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[18:19], v[82:83]
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v54, v8, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v64, 0, v114, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v55, v9, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v65, 0x7ff80000, v51, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v48, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v64, v10, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v49, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[14:15], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v65, v11, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v37, v13, s15
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[34:35], 64
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s9, v[20:21], v[66:67]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[20:21], v[66:67]
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, v32, v16, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v48, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v38, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f64_e32 vcc_lo, v[22:23], v[68:69]
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v35, v15, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, v82, v18, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, 0, v116, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[36:37], 64
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[22:23], v[68:69]
-; GFX10-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v38, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v50, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v51, v13, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[16:17], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, 0, v49, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v83, v19, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[32:33], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[18:19], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v67, v21, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v34, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v35, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v33, v17, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v112, s4
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s4, v[24:25], v[70:71]
-; GFX10-NEXT:    v_cndmask_b32_e32 v113, v69, v23, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v68, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[20:21], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v34, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, 0x7ff80000, v114, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v36, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v37, s13
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[82:83], 64
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[24:25], v[70:71]
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v48, v16, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v49, v17, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, 0x7ff80000, v113, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s5
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s7, v[26:27], v[80:81]
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, v32, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v66, v20, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v33, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v38, v18, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v39, v19, s14
-; GFX10-NEXT:    v_cmp_o_f64_e64 s15, v[26:27], v[80:81]
-; GFX10-NEXT:    v_cndmask_b32_e64 v33, 0, v32, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v71, v25, s4
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s16, v[28:29], v[84:85]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[28:29], v[84:85]
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v34, v21, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v33, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 64
-; GFX10-NEXT:    v_cmp_gt_f64_e64 s17, v[30:31], v[86:87]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[70:71], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v82, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v82, v70, v24, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v19, v83, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v83, 0x7ff80000, v112, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[68:69], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[96:97]
-; GFX10-NEXT:    v_cndmask_b32_e64 v82, 0, v82, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v37, v81, v27, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v80, v26, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[80:81], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[84:85], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[98:99]
-; GFX10-NEXT:    v_cndmask_b32_e64 v113, 0x7ff80000, v37, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, 0, v32, s15
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[100:101]
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, v85, v29, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v84, v28, s16
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[102:103]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[54:55]
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[24:25], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, 0x7ff80000, v115, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, 0, v114, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, v87, v31, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v86, v30, s17
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[86:87], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[64:65]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s15, 0, v[50:51]
-; GFX10-NEXT:    v_cndmask_b32_e64 v117, 0x7ff80000, v116, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, 0, v32, s18
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s16, 0, v[52:53]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[48:49]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s18, 0, v[38:39]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[33:34]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[35:36]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[82:83]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[112:113]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[114:115]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[116:117]
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v22, v68, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v23, v69, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v82, v24, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v83, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[26:27], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v24, v70, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v25, v71, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v54, v8, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v64, v10, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v50, v12, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v48, v16, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v38, v18, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v35, v22, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v82, v24, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v55, v9, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v65, v11, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v51, v13, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s16
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v112, v26, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v113, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[28:29], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v49, v17, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v39, v19, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v26, v26, v80, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v27, v27, v81, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v36, v23, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v83, v25, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v26, v112, v26, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v27, v113, v27, s22
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v114, v28, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v115, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[30:31], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v28, v84, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v29, v85, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v114, v28, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v115, v29, s23
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v116, v30, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v117, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[66:67], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v30, v30, v86, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v31, v31, v87, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v30, v116, v30, s24
-; GFX10-NEXT:    v_cndmask_b32_e64 v31, v117, v31, s24
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v66, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v67, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v33, v20, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v34, v21, s19
+; GFX10-NEXT:    v_max_f64 v[80:81], v[30:31], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s18, v[30:31], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v83, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v84, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v85, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v32, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v33, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v28, v70, 0, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v29, v71, 0x7ff80000, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v26, v66, 0, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v30, v80, 0, s18
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v81, 0x7ff80000, s18
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v16f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_b32 v87, off, s32 offset:8
-; GFX11-NEXT:    scratch_load_b32 v86, off, s32 offset:4
-; GFX11-NEXT:    scratch_load_b32 v85, off, s32 offset:16
-; GFX11-NEXT:    scratch_load_b32 v84, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_b32 v33, off, s32 offset:24
-; GFX11-NEXT:    scratch_load_b32 v32, off, s32 offset:20
-; GFX11-NEXT:    scratch_load_b32 v35, off, s32 offset:32
-; GFX11-NEXT:    scratch_load_b32 v34, off, s32 offset:28
-; GFX11-NEXT:    scratch_load_b32 v37, off, s32 offset:40
-; GFX11-NEXT:    scratch_load_b32 v36, off, s32 offset:36
-; GFX11-NEXT:    scratch_load_b32 v39, off, s32 offset:48
-; GFX11-NEXT:    scratch_load_b32 v38, off, s32 offset:44
-; GFX11-NEXT:    scratch_load_b32 v49, off, s32 offset:56
-; GFX11-NEXT:    scratch_load_b32 v48, off, s32 offset:52
-; GFX11-NEXT:    scratch_load_b32 v51, off, s32 offset:64
-; GFX11-NEXT:    scratch_load_b32 v50, off, s32 offset:60
-; GFX11-NEXT:    scratch_load_b32 v53, off, s32 offset:72
-; GFX11-NEXT:    scratch_load_b32 v52, off, s32 offset:68
-; GFX11-NEXT:    scratch_load_b32 v55, off, s32 offset:80
-; GFX11-NEXT:    scratch_load_b32 v54, off, s32 offset:76
-; GFX11-NEXT:    scratch_load_b32 v65, off, s32 offset:88
-; GFX11-NEXT:    scratch_load_b32 v64, off, s32 offset:84
-; GFX11-NEXT:    scratch_load_b32 v67, off, s32 offset:96
-; GFX11-NEXT:    scratch_load_b32 v66, off, s32 offset:92
-; GFX11-NEXT:    scratch_load_b32 v69, off, s32 offset:104
-; GFX11-NEXT:    scratch_load_b32 v68, off, s32 offset:100
-; GFX11-NEXT:    scratch_load_b32 v71, off, s32 offset:112
-; GFX11-NEXT:    scratch_load_b32 v70, off, s32 offset:108
-; GFX11-NEXT:    scratch_load_b32 v81, off, s32 offset:120
-; GFX11-NEXT:    scratch_load_b32 v80, off, s32 offset:116
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    scratch_load_b32 v83, off, s32 offset:128
-; GFX11-NEXT:    scratch_load_b32 v82, off, s32 offset:124
-; GFX11-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s9, v[0:1], v[86:87]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s11, v[0:1], v[86:87]
-; GFX11-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s10, v[2:3], v[84:85]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s14, v[86:87], 64
-; GFX11-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s0, v[4:5], v[32:33]
-; GFX11-NEXT:    v_cmp_o_f64_e32 vcc_lo, v[4:5], v[32:33]
-; GFX11-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s2, v[6:7], v[34:35]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s12, v[2:3], v[84:85]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[6:7], v[34:35]
-; GFX11-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s4, v[8:9], v[36:37]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[8:9], v[36:37]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, v[84:85], 64
-; GFX11-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s6, v[10:11], v[38:39]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[10:11], v[38:39]
-; GFX11-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s8, v[12:13], v[48:49]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s7, v[12:13], v[48:49]
-; GFX11-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s13, v[14:15], v[50:51]
-; GFX11-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-NEXT:    v_cmp_o_f64_e64 s15, v[16:17], v[52:53]
-; GFX11-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s17, v[18:19], v[54:55]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s18, v[18:19], v[54:55]
-; GFX11-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s19, v[20:21], v[64:65]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s20, v[20:21], v[64:65]
-; GFX11-NEXT:    s_waitcnt vmcnt(9)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s21, v[22:23], v[66:67]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s22, v[22:23], v[66:67]
-; GFX11-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s23, v[24:25], v[68:69]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s24, v[24:25], v[68:69]
-; GFX11-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s25, v[26:27], v[70:71]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s26, v[26:27], v[70:71]
-; GFX11-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s27, v[28:29], v[80:81]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s28, v[28:29], v[80:81]
+; GFX11-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-NEXT:    scratch_load_b32 v49, off, s32 offset:40
+; GFX11-NEXT:    scratch_load_b32 v48, off, s32 offset:36
+; GFX11-NEXT:    scratch_load_b32 v51, off, s32 offset:48
+; GFX11-NEXT:    scratch_load_b32 v50, off, s32 offset:44
+; GFX11-NEXT:    scratch_load_b32 v53, off, s32 offset:56
+; GFX11-NEXT:    scratch_load_b32 v52, off, s32 offset:52
+; GFX11-NEXT:    scratch_load_b32 v55, off, s32 offset:64
+; GFX11-NEXT:    scratch_load_b32 v54, off, s32 offset:60
+; GFX11-NEXT:    scratch_load_b32 v65, off, s32 offset:72
+; GFX11-NEXT:    scratch_load_b32 v64, off, s32 offset:68
+; GFX11-NEXT:    scratch_load_b32 v67, off, s32 offset:80
+; GFX11-NEXT:    scratch_load_b32 v66, off, s32 offset:76
+; GFX11-NEXT:    scratch_load_b32 v69, off, s32 offset:88
+; GFX11-NEXT:    scratch_load_b32 v68, off, s32 offset:84
+; GFX11-NEXT:    scratch_load_b32 v71, off, s32 offset:96
+; GFX11-NEXT:    scratch_load_b32 v70, off, s32 offset:92
+; GFX11-NEXT:    scratch_load_b32 v81, off, s32 offset:104
+; GFX11-NEXT:    scratch_load_b32 v80, off, s32 offset:100
+; GFX11-NEXT:    scratch_load_b32 v83, off, s32 offset:112
+; GFX11-NEXT:    scratch_load_b32 v82, off, s32 offset:108
+; GFX11-NEXT:    scratch_load_b32 v85, off, s32 offset:120
+; GFX11-NEXT:    scratch_load_b32 v84, off, s32 offset:116
+; GFX11-NEXT:    scratch_load_b32 v87, off, s32 offset:128
+; GFX11-NEXT:    scratch_load_b32 v86, off, s32 offset:124
+; GFX11-NEXT:    s_waitcnt vmcnt(30)
+; GFX11-NEXT:    v_max_f64 v[96:97], v[0:1], v[32:33]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33]
+; GFX11-NEXT:    s_waitcnt vmcnt(28)
+; GFX11-NEXT:    v_max_f64 v[32:33], v[2:3], v[34:35]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[34:35]
+; GFX11-NEXT:    s_waitcnt vmcnt(26)
+; GFX11-NEXT:    v_max_f64 v[34:35], v[4:5], v[36:37]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[36:37]
+; GFX11-NEXT:    s_waitcnt vmcnt(24)
+; GFX11-NEXT:    v_max_f64 v[36:37], v[6:7], v[38:39]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[38:39]
+; GFX11-NEXT:    s_waitcnt vmcnt(22)
+; GFX11-NEXT:    v_max_f64 v[38:39], v[8:9], v[48:49]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s3, v[8:9], v[48:49]
+; GFX11-NEXT:    s_waitcnt vmcnt(20)
+; GFX11-NEXT:    v_max_f64 v[48:49], v[10:11], v[50:51]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s4, v[10:11], v[50:51]
+; GFX11-NEXT:    s_waitcnt vmcnt(18)
+; GFX11-NEXT:    v_max_f64 v[50:51], v[12:13], v[52:53]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s5, v[12:13], v[52:53]
+; GFX11-NEXT:    s_waitcnt vmcnt(16)
+; GFX11-NEXT:    v_max_f64 v[52:53], v[14:15], v[54:55]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s6, v[14:15], v[54:55]
+; GFX11-NEXT:    s_waitcnt vmcnt(14)
+; GFX11-NEXT:    v_max_f64 v[54:55], v[16:17], v[64:65]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s7, v[16:17], v[64:65]
+; GFX11-NEXT:    s_waitcnt vmcnt(12)
+; GFX11-NEXT:    v_max_f64 v[64:65], v[18:19], v[66:67]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s8, v[18:19], v[66:67]
+; GFX11-NEXT:    s_waitcnt vmcnt(10)
+; GFX11-NEXT:    v_max_f64 v[66:67], v[20:21], v[68:69]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s9, v[20:21], v[68:69]
+; GFX11-NEXT:    s_waitcnt vmcnt(8)
+; GFX11-NEXT:    v_max_f64 v[68:69], v[22:23], v[70:71]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s10, v[22:23], v[70:71]
+; GFX11-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-NEXT:    v_max_f64 v[70:71], v[24:25], v[80:81]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s11, v[24:25], v[80:81]
+; GFX11-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-NEXT:    v_max_f64 v[80:81], v[26:27], v[82:83]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s12, v[26:27], v[82:83]
+; GFX11-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NEXT:    v_max_f64 v[82:83], v[28:29], v[84:85]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s13, v[28:29], v[84:85]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s29, v[30:31], v[82:83]
-; GFX11-NEXT:    v_cmp_o_f64_e64 vcc_hi, v[30:31], v[82:83]
-; GFX11-NEXT:    v_cndmask_b32_e64 v96, v87, v1, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v101, v86, v0, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v98, v85, v3, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v103, v84, v2, s10
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[0:1], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v97, 0x7ff80000, v96, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v96, 0, v101, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v100, v33, v5, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v102, v35, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v99, 0x7ff80000, v98, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v98, 0, v103, s12
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, v[2:3], 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v101, 0x7ff80000, v100, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v103, 0x7ff80000, v102, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v112, v37, v9, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v114, v39, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v116, v49, v13, s8
-; GFX11-NEXT:    v_cmp_o_f64_e64 s9, v[14:15], v[50:51]
-; GFX11-NEXT:    v_cndmask_b32_e64 v118, v51, v15, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v113, 0x7ff80000, v112, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v115, 0x7ff80000, v114, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v117, 0x7ff80000, v116, s7
-; GFX11-NEXT:    v_cmp_gt_f64_e64 s12, v[16:17], v[52:53]
-; GFX11-NEXT:    v_cndmask_b32_e64 v130, v55, v19, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v132, v65, v21, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v134, v67, v23, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, v69, v25, s23
-; GFX11-NEXT:    v_cndmask_b32_e64 v145, v71, v27, s25
-; GFX11-NEXT:    v_cndmask_b32_e64 v131, 0x7ff80000, v130, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v133, 0x7ff80000, v132, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v135, 0x7ff80000, v134, s22
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, v81, v29, s27
-; GFX11-NEXT:    v_cndmask_b32_e64 v148, v80, v28, s27
-; GFX11-NEXT:    v_cndmask_b32_e64 v147, v83, v31, s29
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v147, 0x7ff80000, v147, vcc_hi
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s10
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[36:37], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v86, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v32, v4, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v87, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v34, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s11
-; GFX11-NEXT:    v_cndmask_b32_e32 v100, 0, v86, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[4:5], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v102, 0, v87, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v84, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v36, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v38, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v48, v12, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v119, 0x7ff80000, v118, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v128, v53, v17, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v112, 0, v84, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v114, 0, v86, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v116, 0, v87, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v50, v14, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v129, 0x7ff80000, v128, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v52, v16, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v54, v18, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v85, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v118, 0, v84, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v64, v20, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v128, 0, v86, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v130, 0, v87, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v66, v22, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v85, 0x7ff80000, v144, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v132, 0, v84, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v68, v24, s23
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, v70, v26, s25
-; GFX11-NEXT:    v_cndmask_b32_e64 v134, 0, v86, s22
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[68:69], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[70:71], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, 0, v87, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, 0x7ff80000, v145, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, 0, v144, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v145, 0x7ff80000, v146, s28
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, 0, v148, s28
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, v82, v30, s29
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[80:81], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[82:83], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[32:33], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, v[34:35], 64
-; GFX11-NEXT:    v_dual_cndmask_b32 v5, v101, v5 :: v_dual_cndmask_b32 v4, v100, v4
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[6:7], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, 0, v146, vcc_hi
-; GFX11-NEXT:    v_cmp_class_f64_e64 s12, v[38:39], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s14, v[48:49], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, v[50:51], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s18, v[52:53], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s20, v[54:55], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s21, v[64:65], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[96:97]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[98:99]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[100:101]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[102:103]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[112:113]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[114:115]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s15, 0, v[116:117]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[118:119]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[128:129]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[130:131]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[132:133]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[134:135]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s25, 0, v[84:85]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s26, 0, v[86:87]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s27, 0, v[144:145]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s28, 0, v[146:147]
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v33, s6
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v103, v7 :: v_dual_cndmask_b32 v6, v102, v6
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[8:9], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v32, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v35, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s9
-; GFX11-NEXT:    v_dual_cndmask_b32 v9, v113, v9 :: v_dual_cndmask_b32 v8, v112, v8
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[10:11], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v34, s8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v37, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s9
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v113, v9, s11
-; GFX11-NEXT:    v_dual_cndmask_b32 v11, v115, v11 :: v_dual_cndmask_b32 v10, v114, v10
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[12:13], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v36, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v39, s12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v112, v8, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v115, v11, s13
-; GFX11-NEXT:    v_dual_cndmask_b32 v13, v117, v13 :: v_dual_cndmask_b32 v12, v116, v12
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[14:15], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v38, s12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v49, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v114, v10, s13
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v117, v13, s15
-; GFX11-NEXT:    v_dual_cndmask_b32 v15, v119, v15 :: v_dual_cndmask_b32 v14, v118, v14
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[16:17], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v48, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v51, s16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v116, v12, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v119, v15, s17
-; GFX11-NEXT:    v_dual_cndmask_b32 v17, v129, v17 :: v_dual_cndmask_b32 v16, v128, v16
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[18:19], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v50, s16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v17, v53, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v118, v14, s17
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v129, v17, s19
-; GFX11-NEXT:    v_dual_cndmask_b32 v19, v131, v19 :: v_dual_cndmask_b32 v18, v130, v18
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[20:21], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v16, v52, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v19, v55, s20
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v128, v16, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v131, v19, s22
-; GFX11-NEXT:    v_dual_cndmask_b32 v21, v133, v21 :: v_dual_cndmask_b32 v20, v132, v20
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v18, v54, s20
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v21, v65, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v130, v18, s22
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v133, v21, s23
-; GFX11-NEXT:    v_dual_cndmask_b32 v23, v135, v23 :: v_dual_cndmask_b32 v22, v134, v22
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[24:25], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v20, v64, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v132, v20, s23
-; GFX11-NEXT:    v_dual_cndmask_b32 v25, v85, v25 :: v_dual_cndmask_b32 v24, v84, v24
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[26:27], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v25, v25, v69, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v25, v85, v25, s25
-; GFX11-NEXT:    v_dual_cndmask_b32 v27, v87, v27 :: v_dual_cndmask_b32 v26, v86, v26
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[28:29], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v24, v68, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v27, v27, v71, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v84, v24, s25
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v27, v87, v27, s26
-; GFX11-NEXT:    v_dual_cndmask_b32 v29, v145, v29 :: v_dual_cndmask_b32 v28, v144, v28
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[30:31], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v26, v26, v70, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v29, v29, v81, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v26, v86, v26, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v29, v145, v29, s27
-; GFX11-NEXT:    v_dual_cndmask_b32 v31, v147, v31 :: v_dual_cndmask_b32 v30, v146, v30
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[66:67], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v28, v28, v80, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v31, v31, v83, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v28, v144, v28, s27
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v31, v147, v31, s28
-; GFX11-NEXT:    v_dual_cndmask_b32 v23, v23, v67 :: v_dual_cndmask_b32 v22, v22, v66
-; GFX11-NEXT:    v_cndmask_b32_e64 v30, v30, v82, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, v135, v23, s24
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v134, v22, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v30, v146, v30, s28
+; GFX11-NEXT:    v_max_f64 v[84:85], v[30:31], v[86:87]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s14, v[30:31], v[86:87]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v32, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v33, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v34, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v35, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v36, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v37, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v8, v38, 0, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v39, 0x7ff80000, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v48, 0, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v49, 0x7ff80000, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v50, 0, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v51, 0x7ff80000, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v52, 0, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v53, 0x7ff80000, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v16, v54, 0, s7
+; GFX11-NEXT:    v_cndmask_b32_e64 v17, v55, 0x7ff80000, s7
+; GFX11-NEXT:    v_cndmask_b32_e64 v18, v64, 0, s8
+; GFX11-NEXT:    v_cndmask_b32_e64 v19, v65, 0x7ff80000, s8
+; GFX11-NEXT:    v_cndmask_b32_e64 v20, v66, 0, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v21, v67, 0x7ff80000, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v22, v68, 0, s10
+; GFX11-NEXT:    v_cndmask_b32_e64 v23, v69, 0x7ff80000, s10
+; GFX11-NEXT:    v_cndmask_b32_e64 v24, v70, 0, s11
+; GFX11-NEXT:    v_cndmask_b32_e64 v25, v71, 0x7ff80000, s11
+; GFX11-NEXT:    v_cndmask_b32_e64 v26, v80, 0, s12
+; GFX11-NEXT:    v_cndmask_b32_e64 v27, v81, 0x7ff80000, s12
+; GFX11-NEXT:    v_cndmask_b32_e64 v28, v82, 0, s13
+; GFX11-NEXT:    v_cndmask_b32_e64 v29, v83, 0x7ff80000, s13
+; GFX11-NEXT:    v_cndmask_b32_e64 v30, v84, 0, s14
+; GFX11-NEXT:    v_cndmask_b32_e64 v31, v85, 0x7ff80000, s14
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v16f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index 95d351e8f1fad2984f6dbcf9a744900a711e8304..e00ebff751c73e69a82f808d1bac078139577a43 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -443,28 +443,14 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; GFX8-NEXT:    v_min_f16_e32 v4, v3, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX8-NEXT:    v_min_f16_e32 v3, v0, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -543,26 +529,9 @@ define <2 x half> @v_minimum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) {
 ; GFX8-LABEL: v_minimum_v2f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f16__nnan:
@@ -608,13 +577,11 @@ define <2 x half> @v_minimum_v2f16__nsz(<2 x half> %src0, <2 x half> %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; GFX8-NEXT:    v_min_f16_e32 v4, v3, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX8-NEXT:    v_min_f16_e32 v3, v0, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
@@ -696,14 +663,9 @@ define <2 x half> @v_minimum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1)
 ; GFX8-LABEL: v_minimum_v2f16__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f16__nnan_nsz:
@@ -750,31 +712,15 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX8-NEXT:    s_lshr_b32 s6, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, s7, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
+; GFX8-NEXT:    v_min_f16_e32 v1, s7, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s7, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_lt_f16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX8-NEXT:    v_min_f16_e32 v3, s4, v1
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
index 1da2647fbd605b0f8d3800d80a9e622d05de573d..e056682051aa452440fa18810c71bbdac5d305a9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
@@ -495,167 +495,73 @@ define <2 x float> @v_minimum_v2f32(<2 x float> %src0, <2 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v2f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v1, v3
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX7-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX8-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX940-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v2, v1, v3
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v5, v1, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v4, v0, v2 :: v_dual_min_f32 v5, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f32:
@@ -676,136 +582,42 @@ define <2 x float> @v_minimum_v2f32__nnan(<2 x float> %src0, <2 x float> %src1)
 ; GFX7-LABEL: v_minimum_v2f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v0, v2
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v1, v3
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f32__nnan:
@@ -826,11 +638,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v2f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v1, v3
+; GFX7-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -838,13 +650,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX8-LABEL: v_minimum_v2f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -852,13 +662,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX9-LABEL: v_minimum_v2f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v2, v1, v3
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -866,16 +674,12 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX940-LABEL: v_minimum_v2f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX940-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_min_f32_e32 v2, v1, v3
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
@@ -884,11 +688,9 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX10-LABEL: v_minimum_v2f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v4, v0, v2
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
+; GFX10-NEXT:    v_min_f32_e32 v5, v1, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
@@ -897,12 +699,9 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) {
 ; GFX11-LABEL: v_minimum_v2f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v4, v0, v2 :: v_dual_min_f32 v5, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v3
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo
@@ -926,55 +725,42 @@ define <2 x float> @v_minimum_v2f32__nnan_nsz(<2 x float> %src0, <2 x float> %sr
 ; GFX7-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v0, v2
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v1, v3
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f32__nnan_nsz:
@@ -996,28 +782,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, s5, v0
+; GFX7-NEXT:    v_min_f32_e32 v1, s5, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s7, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, s4, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_min_f32_e32 v3, s4, v0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v3, s4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s6, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use v[0:1]
 ; GFX7-NEXT:    ;;#ASMEND
@@ -1027,30 +799,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s7
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
+; GFX8-NEXT:    v_min_f32_e32 v1, s5, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v3, s4, v0
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
@@ -1060,30 +816,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
-; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
+; GFX9-NEXT:    v_min_f32_e32 v1, s5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s5, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, s4, v0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v[0:1]
 ; GFX9-NEXT:    ;;#ASMEND
@@ -1093,40 +833,15 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s3
-; GFX940-NEXT:    v_mov_b32_e32 v1, s1
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v1, s1, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s1, v0
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v3, s0, v0
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
@@ -1135,28 +850,12 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX10-LABEL: s_minimum_v2f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s5
-; GFX10-NEXT:    v_cmp_lt_f32_e64 vcc_lo, s5, s7
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    v_cmp_class_f32_e64 s8, s5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, s7, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e64 vcc_lo, s4, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s6, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e64 v0, s5, s7
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s5, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX10-NEXT:    v_min_f32_e64 v2, s4, s6
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s4, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s5, s8
-; GFX10-NEXT:    v_cmp_class_f32_e64 s5, s4, 32
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v2, s4, s5
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s7, 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s7, s4
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s6, s4
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v[0:1]
 ; GFX10-NEXT:    ;;#ASMEND
@@ -1165,32 +864,13 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) {
 ; GFX11-LABEL: s_minimum_v2f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT:    v_cmp_lt_f32_e64 vcc_lo, s1, s3
-; GFX11-NEXT:    v_cmp_class_f32_e64 s4, s1, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, s3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e64 vcc_lo, s0, s2
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_min_f32_e64 v0, s1, s3
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s1, s3
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX11-NEXT:    v_min_f32_e64 v2, s0, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s1, s4
-; GFX11-NEXT:    v_cmp_class_f32_e64 s1, s0, 32
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v2, s0, s1
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s3, 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s2, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s2, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v[0:1]
 ; GFX11-NEXT:    ;;#ASMEND
@@ -1218,227 +898,92 @@ define <3 x float> @v_minimum_v3f32(<3 x float> %src0, <3 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v3f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v6, v0, v3
+; GFX7-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v1, v4
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX7-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v2, v5
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX7-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX8-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX8-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX940-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v3, v1, v4
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
+; GFX940-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v7, v1, v4
+; GFX10-NEXT:    v_min_f32_e32 v8, v2, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v6, v0, v3 :: v_dual_min_f32 v7, v1, v4
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f32:
@@ -1460,184 +1005,48 @@ define <3 x float> @v_minimum_v3f32__nnan(<3 x float> %src0, <3 x float> %src1)
 ; GFX7-LABEL: v_minimum_v3f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v6, v0, v3
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v1, v4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v2, v5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX9-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX940-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX10-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v3 :: v_dual_min_f32 v1, v1, v4
+; GFX11-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f32__nnan:
@@ -1659,14 +1068,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v3f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v6, v0, v3
+; GFX7-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v1, v4
+; GFX7-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v2, v5
+; GFX7-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -1674,17 +1083,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX8-LABEL: v_minimum_v3f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX8-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1692,17 +1098,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX9-LABEL: v_minimum_v3f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v1, v4
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1710,22 +1113,16 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX940-LABEL: v_minimum_v3f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX940-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_min_f32_e32 v3, v1, v4
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v3, v2, v5
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
@@ -1734,13 +1131,10 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX10-LABEL: v_minimum_v3f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v6, v0, v3
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
+; GFX10-NEXT:    v_min_f32_e32 v7, v1, v4
+; GFX10-NEXT:    v_min_f32_e32 v8, v2, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
@@ -1751,17 +1145,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) {
 ; GFX11-LABEL: v_minimum_v3f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v6, v0, v3 :: v_dual_min_f32 v7, v1, v4
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1784,67 +1175,48 @@ define <3 x float> @v_minimum_v3f32__nnan_nsz(<3 x float> %src0, <3 x float> %sr
 ; GFX7-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v0, v3
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v1, v4
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v2, v5
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX9-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX940-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v3
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v4
+; GFX10-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v3 :: v_dual_min_f32 v1, v1, v4
+; GFX11-NEXT:    v_min_f32_e32 v2, v2, v5
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f32__nnan_nsz:
@@ -1866,292 +1238,111 @@ define <4 x float> @v_minimum_v4f32(<4 x float> %src0, <4 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v4f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v0, v4
+; GFX7-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v1, v5
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v2, v6
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX7-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v3, v7
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX7-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
+; GFX940-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v4, v1, v5
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
+; GFX940-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
+; GFX940-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v9, v1, v5
+; GFX10-NEXT:    v_min_f32_e32 v4, v2, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v3, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v8, v0, v4 :: v_dual_min_f32 v9, v1, v5
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v4, v2, v6
+; GFX11-NEXT:    v_dual_min_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f32:
@@ -2174,236 +1365,53 @@ define <4 x float> @v_minimum_v4f32__nnan(<4 x float> %src0, <4 x float> %src1)
 ; GFX7-LABEL: v_minimum_v4f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v0, v4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v1, v5
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v2, v6
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v3, v7
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX8-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX8-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX9-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX9-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX940-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX940-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX10-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX10-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v7, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v1, v1, v5
+; GFX11-NEXT:    v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f32__nnan:
@@ -2426,17 +1434,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v4f32__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v0, v4
+; GFX7-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v1, v5
+; GFX7-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v2, v6
+; GFX7-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v4, v3, v7
+; GFX7-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
@@ -2444,21 +1452,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX8-LABEL: v_minimum_v4f32__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX8-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -2466,21 +1470,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX9-LABEL: v_minimum_v4f32__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2488,28 +1488,20 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX940-LABEL: v_minimum_v4f32__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
+; GFX940-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX940-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_min_f32_e32 v4, v1, v5
+; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
+; GFX940-NEXT:    v_min_f32_e32 v4, v3, v7
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
@@ -2518,44 +1510,35 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) {
 ; GFX10-LABEL: v_minimum_v4f32__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v0, v4
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
+; GFX10-NEXT:    v_min_f32_e32 v9, v1, v5
+; GFX10-NEXT:    v_min_f32_e32 v4, v2, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v7, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v3, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f32__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v8, v0, v4 :: v_dual_min_f32 v9, v1, v5
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v7, v3, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v4, v2, v6
+; GFX11-NEXT:    v_dual_min_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f32__nsz:
@@ -2578,79 +1561,53 @@ define <4 x float> @v_minimum_v4f32__nnan_nsz(<4 x float> %src0, <4 x float> %sr
 ; GFX7-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v0, v4
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v1, v5
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v2, v6
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v3, v7
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX8-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX8-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX9-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX9-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX940-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX940-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX940-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX10-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX10-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX10-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f32__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v1, v1, v5
+; GFX11-NEXT:    v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f32__nnan_nsz:
@@ -2673,551 +1630,185 @@ define <8 x float> @v_minimum_v8f32(<8 x float> %src0, <8 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v8f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v0, v8
+; GFX7-NEXT:    v_min_f32_e32 v16, v0, v8
 ; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v1, v9
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v1, v9
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v2, v10
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v2, v10
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v3, v11
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v3, v11
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v4, v12
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v4, v12
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v5, v13
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v5, v13
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v6, v14
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v6, v14
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v8, v7, v15
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX7-NEXT:    v_min_f32_e32 v8, v7, v15
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v8f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v16, v0, v8
 ; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v1, v9
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v2, v10
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v3, v11
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v4, v12
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v5, v13
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v6, v14
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX8-NEXT:    v_min_f32_e32 v8, v7, v15
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v8f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v16, v0, v8
 ; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v1, v9
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v2, v10
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v3, v11
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v4, v12
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v5, v13
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v6, v14
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX9-NEXT:    v_min_f32_e32 v8, v7, v15
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v8f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v8
+; GFX940-NEXT:    v_min_f32_e32 v16, v0, v8
 ; GFX940-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v1, v9
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v10
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v2, v10
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v11
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v3, v11
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v12
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v4, v12
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v13
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v5, v13
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v14
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v6, v14
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v15
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
+; GFX940-NEXT:    v_min_f32_e32 v8, v7, v15
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v8f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v16, v0, v8
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v17, v1, v9
+; GFX10-NEXT:    v_min_f32_e32 v8, v2, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v12, v4, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v9, v3, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v10, v7, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v4, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v9, v5, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v8, v6, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v8f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v16, v0, v8 :: v_dual_min_f32 v17, v1, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v12, v4, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v9, v3, v11 :: v_dual_min_f32 v8, v2, v10
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v10, v7, v15
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v8, v4, v12 :: v_dual_cndmask_b32 v3, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v9, v5, v13 :: v_dual_cndmask_b32 v4, 0x7fc00000, v8
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_min_f32 v8, v6, v14 :: v_dual_cndmask_b32 v5, 0x7fc00000, v9
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v8f32:
@@ -3244,1071 +1835,371 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v16f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v32, v0, v16
-; GFX7-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v16, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v1, v17
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v16
+; GFX7-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX7-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX7-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v2, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v3, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v4, v20
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v20, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v5, v21
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v21, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v6, v22
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v22, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v7, v23
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v23, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v8, v24
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v24, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v9, v25
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v25, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v10, v26
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v26, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v11, v27
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v27, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v12, v28
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v28, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v13, v29
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v29, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v14, v30
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v30, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v18
+; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX7-NEXT:    v_min_f32_e32 v18, v13, v29
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX7-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX7-NEXT:    v_min_f32_e32 v3, v3, v19
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX7-NEXT:    v_min_f32_e32 v4, v4, v20
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX7-NEXT:    v_min_f32_e32 v5, v5, v21
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX7-NEXT:    v_min_f32_e32 v6, v6, v22
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX7-NEXT:    v_min_f32_e32 v7, v7, v23
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX7-NEXT:    v_min_f32_e32 v8, v8, v24
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX7-NEXT:    v_min_f32_e32 v9, v9, v25
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX7-NEXT:    v_min_f32_e32 v10, v10, v26
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX7-NEXT:    v_min_f32_e32 v11, v11, v27
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX7-NEXT:    v_min_f32_e32 v12, v12, v28
+; GFX7-NEXT:    v_min_f32_e32 v19, v14, v30
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX7-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX7-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_min_f32_e32 v18, v15, v16
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX7-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v16, v15, v17
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v16f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v16, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v16
+; GFX8-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX8-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v18, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v19, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v20, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v21, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v22, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v23, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v24, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v25, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v26, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v27, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v28, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v29, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v30, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX8-NEXT:    v_min_f32_e32 v1, v1, v17
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX8-NEXT:    v_min_f32_e32 v2, v2, v18
+; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX8-NEXT:    v_min_f32_e32 v18, v13, v29
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX8-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX8-NEXT:    v_min_f32_e32 v3, v3, v19
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX8-NEXT:    v_min_f32_e32 v4, v4, v20
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX8-NEXT:    v_min_f32_e32 v5, v5, v21
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX8-NEXT:    v_min_f32_e32 v6, v6, v22
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX8-NEXT:    v_min_f32_e32 v7, v7, v23
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX8-NEXT:    v_min_f32_e32 v8, v8, v24
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX8-NEXT:    v_min_f32_e32 v9, v9, v25
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX8-NEXT:    v_min_f32_e32 v10, v10, v26
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX8-NEXT:    v_min_f32_e32 v11, v11, v27
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX8-NEXT:    v_min_f32_e32 v12, v12, v28
+; GFX8-NEXT:    v_min_f32_e32 v19, v14, v30
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX8-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX8-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_min_f32_e32 v18, v15, v16
+; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
-; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v16f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v31, 0x7fc00000
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v32, v31, v32, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v16, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[16:17], v0, v16
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v16
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX9-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v18, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v19
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v19, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v20
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v20, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v21
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v21, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v22
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v22, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v23
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v23, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v24
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v24, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v25
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v25, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v26
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v26, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v27
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v27, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v28
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v28, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v29
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v29, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v30
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v30, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v17
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
+; GFX9-NEXT:    v_min_f32_e32 v2, v2, v18
+; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
+; GFX9-NEXT:    v_min_f32_e32 v18, v13, v29
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[28:29], v13, v29
+; GFX9-NEXT:    v_writelane_b32 v31, s31, 1
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
+; GFX9-NEXT:    v_min_f32_e32 v3, v3, v19
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
+; GFX9-NEXT:    v_min_f32_e32 v4, v4, v20
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
+; GFX9-NEXT:    v_min_f32_e32 v5, v5, v21
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[12:13], v6, v22
+; GFX9-NEXT:    v_min_f32_e32 v6, v6, v22
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[14:15], v7, v23
+; GFX9-NEXT:    v_min_f32_e32 v7, v7, v23
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v24
+; GFX9-NEXT:    v_min_f32_e32 v8, v8, v24
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v25
+; GFX9-NEXT:    v_min_f32_e32 v9, v9, v25
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v26
+; GFX9-NEXT:    v_min_f32_e32 v10, v10, v26
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v27
+; GFX9-NEXT:    v_min_f32_e32 v11, v11, v27
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[26:27], v12, v28
+; GFX9-NEXT:    v_min_f32_e32 v12, v12, v28
+; GFX9-NEXT:    v_min_f32_e32 v19, v14, v30
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v18, s[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v17, v12, s[26:27]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v17, v19, s[30:31]
+; GFX9-NEXT:    v_readlane_b32 s31, v31, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v31, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_min_f32_e32 v18, v15, v16
+; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
-; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v31, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v16f32:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v16
 ; GFX940-NEXT:    v_mov_b32_e32 v32, 0x7fc00000
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v16, v0, vcc
+; GFX940-NEXT:    v_min_f32_e32 v33, v0, v16
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v32, v33, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v16, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v17
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v1, vcc
+; GFX940-NEXT:    v_min_f32_e32 v34, v1, v17
+; GFX940-NEXT:    v_min_f32_e32 v35, v2, v18
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v32, v33, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v17, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
+; GFX940-NEXT:    v_min_f32_e32 v36, v3, v19
+; GFX940-NEXT:    v_min_f32_e32 v37, v4, v20
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v32, v34, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v18, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v19
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
+; GFX940-NEXT:    v_min_f32_e32 v38, v5, v21
+; GFX940-NEXT:    v_min_f32_e32 v39, v6, v22
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v32, v35, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v19, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v20
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
+; GFX940-NEXT:    v_min_f32_e32 v48, v7, v23
+; GFX940-NEXT:    v_min_f32_e32 v49, v8, v24
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v32, v36, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v4, v20
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v20, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v21
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
+; GFX940-NEXT:    v_min_f32_e32 v50, v9, v25
+; GFX940-NEXT:    v_min_f32_e32 v51, v10, v26
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v32, v37, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v5, v21
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v21, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v22
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
+; GFX940-NEXT:    v_min_f32_e32 v52, v11, v27
+; GFX940-NEXT:    v_min_f32_e32 v53, v12, v28
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v32, v38, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v6, v22
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v22, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v23
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
+; GFX940-NEXT:    v_min_f32_e32 v54, v13, v29
+; GFX940-NEXT:    v_min_f32_e32 v55, v14, v30
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v32, v39, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v7, v23
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v23, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v24
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_min_f32_e32 v16, v15, v31
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v32, v48, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v8, v24
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v8, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v24, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v25
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v8, v32, v49, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v9, v25
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v9, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v25, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v26
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v32, v50, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v10, v26
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v10, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v26, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v27
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v32, v51, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v11, v27
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v11, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v27, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v28
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v32, v52, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v12, v28
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v12, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v28, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v29
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v12, v32, v53, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v13, v29
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v13, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v29, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v30
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v32, v54, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v14, v30
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v14, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v30, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v31
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v14, v32, v55, vcc
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v15, v31
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v32, v16, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v15, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v31, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v32, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v16f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v16
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v18, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v19, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v20, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v22, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v23, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v24, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v25, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v26, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v27, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v28, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v29, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v30, v14, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v32, v0, v16
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v33, v1, v17
+; GFX10-NEXT:    v_min_f32_e32 v34, v2, v18
+; GFX10-NEXT:    v_min_f32_e32 v35, v3, v19
+; GFX10-NEXT:    v_min_f32_e32 v36, v4, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v37, v5, v21
+; GFX10-NEXT:    v_min_f32_e32 v38, v6, v22
+; GFX10-NEXT:    v_min_f32_e32 v39, v7, v23
+; GFX10-NEXT:    v_min_f32_e32 v48, v8, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v49, v9, v25
+; GFX10-NEXT:    v_min_f32_e32 v50, v10, v26
+; GFX10-NEXT:    v_min_f32_e32 v51, v11, v27
+; GFX10-NEXT:    v_min_f32_e32 v52, v12, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v53, v13, v29
+; GFX10-NEXT:    v_min_f32_e32 v54, v14, v30
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v8, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v9, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v10, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v11, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v12, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v13, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v14, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v16, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v17, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v18, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v19, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v20, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v21, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v22, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v23, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v24, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v25, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v26, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v27, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v28, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v29, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v30, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v16, v15, v31
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v15, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v31, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v16f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v0, v16
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v1, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v34, v18, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v35, v19, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v36, v20, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v37, v21, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e32 v38, v22, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v39, v23, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v24
-; GFX11-NEXT:    v_cndmask_b32_e32 v48, v24, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v25
-; GFX11-NEXT:    v_cndmask_b32_e32 v49, v25, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v26
-; GFX11-NEXT:    v_cndmask_b32_e32 v50, v26, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v27
-; GFX11-NEXT:    v_cndmask_b32_e32 v51, v27, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v28
-; GFX11-NEXT:    v_cndmask_b32_e32 v52, v28, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v29
-; GFX11-NEXT:    v_cndmask_b32_e32 v53, v29, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v30
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, v30, v14, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v32, v0, v16 :: v_dual_min_f32 v33, v1, v17
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v34, v2, v18 :: v_dual_min_f32 v35, v3, v19
+; GFX11-NEXT:    v_dual_min_f32 v36, v4, v20 :: v_dual_min_f32 v37, v5, v21
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v1, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v54, v14, v30
+; GFX11-NEXT:    v_dual_min_f32 v38, v6, v22 :: v_dual_min_f32 v39, v7, v23
+; GFX11-NEXT:    v_dual_min_f32 v48, v8, v24 :: v_dual_min_f32 v49, v9, v25
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v2, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo
+; GFX11-NEXT:    v_dual_min_f32 v50, v10, v26 :: v_dual_min_f32 v51, v11, v27
+; GFX11-NEXT:    v_dual_min_f32 v52, v12, v28 :: v_dual_min_f32 v53, v13, v29
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v3, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v4, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v5, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v7, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v8, v24
-; GFX11-NEXT:    v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v9, v25
-; GFX11-NEXT:    v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v10, v26
-; GFX11-NEXT:    v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v11, v27
-; GFX11-NEXT:    v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v12, v28
-; GFX11-NEXT:    v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v13, v29
-; GFX11-NEXT:    v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v14, v30
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v8, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v11, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v12, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v13, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v14, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v16, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v17, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v18, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v19, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v20, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v21, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v22, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v23, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v24, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v25, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v26, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v27, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v28, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v29, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v30, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v34, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v35, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v36, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v37, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v38, v6, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v31
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v39, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v48, v8, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v16, v15, v31
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v15, v31
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v49, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v15, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v52, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v31, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v15, v31, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v16f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
index 7013c60bada55dff9e733d75fc2a711e824a37b3..37fe2e958e62bfed728f9c01d8a039fa78747759 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
@@ -530,221 +530,86 @@ define <2 x double> @v_minimum_v2f64(<2 x double> %src0, <2 x double> %src1) {
 ; GFX7-LABEL: v_minimum_v2f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[0:1], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v7, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v10, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_mov_b32_e32 v10, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v10, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[4:5], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v10, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v8, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[6:7]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v6, v2, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0x7ff80000, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v10, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, v12, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, v13, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s8
+; GFX10-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v5, v1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v6, v2, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, 0x7ff80000, v8, s1
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v10, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, 0, v13, s2
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[6:7], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[10:11]
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, 0, v12, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s3, 0, v[8:9]
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s3
+; GFX11-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[6:7]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f64:
@@ -765,182 +630,43 @@ define <2 x double> @v_minimum_v2f64__nnan(<2 x double> %src0, <2 x double> %src
 ; GFX7-LABEL: v_minimum_v2f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[6:7], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[4:5], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 32
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[0:1]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s8
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[6:7], 32
-; GFX11-NEXT:    v_dual_cndmask_b32 v9, v5, v1 :: v_dual_cndmask_b32 v8, v4, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v6, v2, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s3, 0, v[8:9]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f64__nnan:
@@ -961,111 +687,86 @@ define <2 x double> @v_minimum_v2f64__nsz(<2 x double> %src0, <2 x double> %src1
 ; GFX7-LABEL: v_minimum_v2f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[8:9]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v8, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[6:7]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v6, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s6
+; GFX10-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[6:7]
-; GFX11-NEXT:    v_dual_cndmask_b32 v8, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
+; GFX11-NEXT:    v_min_f64 v[8:9], v[0:1], v[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[2:3], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[6:7]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v6, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v8, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f64__nsz:
@@ -1086,69 +787,43 @@ define <2 x double> @v_minimum_v2f64__nnan_nsz(<2 x double> %src0, <2 x double>
 ; GFX7-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[6:7]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s4
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[6:7]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[6:7]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f64__nnan_nsz:
@@ -1170,61 +845,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 32
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX7-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX7-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 32
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX7-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX7-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX7-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX7-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX7-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX7-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX7-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX7-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX7-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX7-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX7-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX7-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX7-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX7-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 32
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 32
-; GFX7-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX7-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX7-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX7-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX7-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX7-NEXT:    v_mov_b32_e32 v5, s9
+; GFX7-NEXT:    v_min_f64 v[2:3], s[6:7], v[0:1]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX7-NEXT:    v_min_f64 v[0:1], s[4:5], v[4:5]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX7-NEXT:    ;;#ASMSTART
-; GFX7-NEXT:    ; use s[4:7]
+; GFX7-NEXT:    ; use v[0:3]
 ; GFX7-NEXT:    ;;#ASMEND
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1232,61 +866,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s11
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 32
-; GFX8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX8-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX8-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX8-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 32
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX8-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX8-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX8-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX8-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX8-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX8-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX8-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX8-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX8-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX8-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX8-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX8-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX8-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 32
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 32
-; GFX8-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX8-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX8-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX8-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX8-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    v_mov_b32_e32 v5, s9
+; GFX8-NEXT:    v_min_f64 v[2:3], s[6:7], v[0:1]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX8-NEXT:    v_min_f64 v[0:1], s[4:5], v[4:5]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX8-NEXT:    ;;#ASMSTART
-; GFX8-NEXT:    ; use s[4:7]
+; GFX8-NEXT:    ; use v[0:3]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1294,61 +887,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
+; GFX9-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s11
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[18:19], s[10:11], 32
-; GFX9-NEXT:    v_mov_b32_e32 v0, s8
-; GFX9-NEXT:    v_mov_b32_e32 v1, s9
-; GFX9-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s16, s7, s11
-; GFX9-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s15, s16, 0x7ff80000
-; GFX9-NEXT:    s_and_b64 s[16:17], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s14, s6, s10
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[16:17], s[6:7], 32
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s14, s14, 0
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[20:21], s[14:15], 0
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX9-NEXT:    s_and_b64 s[12:13], s[18:19], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX9-NEXT:    s_and_b64 s[12:13], s[20:21], exec
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s15
-; GFX9-NEXT:    s_and_b64 s[12:13], s[16:17], exec
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1]
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX9-NEXT:    s_and_b64 s[16:17], s[18:19], exec
-; GFX9-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX9-NEXT:    s_and_b64 s[10:11], s[20:21], exec
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s14
-; GFX9-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s14, s5, s9
-; GFX9-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s11, s14, 0x7ff80000
-; GFX9-NEXT:    s_and_b64 s[14:15], vcc, exec
-; GFX9-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[14:15], s[4:5], 32
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[12:13], s[8:9], 32
-; GFX9-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[16:17], s[10:11], 0
-; GFX9-NEXT:    s_and_b64 s[18:19], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[18:19], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX9-NEXT:    s_and_b64 s[18:19], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[14:15], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_and_b64 s[8:9], s[16:17], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    v_mov_b32_e32 v5, s9
+; GFX9-NEXT:    v_min_f64 v[2:3], s[6:7], v[0:1]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[6:7], v[0:1]
+; GFX9-NEXT:    v_min_f64 v[0:1], s[4:5], v[4:5]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[4:5]
 ; GFX9-NEXT:    ;;#ASMSTART
-; GFX9-NEXT:    ; use s[4:7]
+; GFX9-NEXT:    ; use v[0:3]
 ; GFX9-NEXT:    ;;#ASMEND
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1356,179 +908,52 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1)
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, s[2:3], v[0:1]
-; GFX940-NEXT:    s_and_b64 s[8:9], vcc, exec
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[8:9], s[2:3], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s12, s3, s7
-; GFX940-NEXT:    s_and_b64 s[10:11], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s11, s12, 0x7ff80000
-; GFX940-NEXT:    s_and_b64 s[12:13], vcc, exec
-; GFX940-NEXT:    s_cselect_b32 s10, s2, s6
-; GFX940-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[12:13], s[2:3], 32
-; GFX940-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX940-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[14:15], s[6:7], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[8:9], s[10:11], 0
-; GFX940-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX940-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX940-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX940-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s7, s3, s11
-; GFX940-NEXT:    s_and_b64 s[12:13], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s11, s2, s10
-; GFX940-NEXT:    s_and_b64 s[2:3], s[14:15], exec
+; GFX940-NEXT:    v_min_f64 v[2:3], s[2:3], v[0:1]
+; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[2:3], v[0:1]
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
-; GFX940-NEXT:    s_cselect_b32 s6, s6, s11
-; GFX940-NEXT:    s_and_b64 s[2:3], s[8:9], exec
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s6, s6, s10
-; GFX940-NEXT:    s_and_b64 s[2:3], vcc, exec
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[2:3], s[0:1], v[0:1]
-; GFX940-NEXT:    s_cselect_b32 s10, s1, s5
-; GFX940-NEXT:    s_and_b64 s[8:9], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s9, s10, 0x7ff80000
-; GFX940-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX940-NEXT:    s_cselect_b32 s8, s0, s4
-; GFX940-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[10:11], s[0:1], 32
-; GFX940-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX940-NEXT:    s_and_b64 s[12:13], s[10:11], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[12:13], s[4:5], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[8:9], 0
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s9
-; GFX940-NEXT:    s_and_b64 s[14:15], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX940-NEXT:    s_and_b64 s[14:15], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s5, s1, s9
-; GFX940-NEXT:    s_and_b64 s[10:11], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s9, s0, s8
-; GFX940-NEXT:    s_and_b64 s[0:1], s[12:13], exec
-; GFX940-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX940-NEXT:    s_and_b64 s[0:1], s[2:3], exec
-; GFX940-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX940-NEXT:    v_min_f64 v[4:5], s[0:1], v[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[0:1], v[0:1]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; use s[4:7]
+; GFX940-NEXT:    ; use v[0:3]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: s_minimum_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s12, s[6:7], s[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s14, s[6:7], s[10:11]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s15, s[6:7], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s16, s[10:11], 32
-; GFX10-NEXT:    v_cmp_o_f64_e64 s18, s[4:5], s[8:9]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s19, s[4:5], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s20, s[8:9], 32
-; GFX10-NEXT:    s_and_b32 s13, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s13, s7, s11
-; GFX10-NEXT:    s_and_b32 s17, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s13, s13, 0x7ff80000
-; GFX10-NEXT:    s_and_b32 s12, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s12, s6, s10
-; GFX10-NEXT:    s_and_b32 s14, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s12, s12, 0
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s17, s[4:5], s[8:9]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, s[12:13], 0
-; GFX10-NEXT:    s_and_b32 s21, s15, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s13
-; GFX10-NEXT:    s_and_b32 s21, s16, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s11, s7
-; GFX10-NEXT:    s_and_b32 s11, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s13
-; GFX10-NEXT:    s_and_b32 s11, s15, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s12
-; GFX10-NEXT:    s_and_b32 s11, s16, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX10-NEXT:    s_and_b32 s10, s14, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s12
-; GFX10-NEXT:    s_and_b32 s10, s17, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s5, s9
-; GFX10-NEXT:    s_and_b32 s11, s18, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s11, s10, 0x7ff80000
-; GFX10-NEXT:    s_and_b32 s10, s17, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX10-NEXT:    s_and_b32 s12, s18, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX10-NEXT:    s_and_b32 s13, s19, exec_lo
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, s[10:11], 0
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_and_b32 s13, s20, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX10-NEXT:    s_and_b32 s9, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_and_b32 s9, s19, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX10-NEXT:    s_and_b32 s9, s20, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_and_b32 s8, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX10-NEXT:    v_min_f64 v[0:1], s[6:7], s[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, s[6:7], s[10:11]
+; GFX10-NEXT:    v_min_f64 v[4:5], s[4:5], s[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, s[4:5], s[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v1, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, s4
 ; GFX10-NEXT:    ;;#ASMSTART
-; GFX10-NEXT:    ; use s[4:7]
+; GFX10-NEXT:    ; use v[0:3]
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: s_minimum_v2f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s8, s[2:3], s[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s10, s[2:3], s[6:7]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, s[2:3], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s12, s[6:7], 32
-; GFX11-NEXT:    v_cmp_o_f64_e64 s14, s[0:1], s[4:5]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s15, s[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, s[4:5], 32
-; GFX11-NEXT:    s_and_b32 s9, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s9, s3, s7
-; GFX11-NEXT:    s_and_b32 s13, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s9, s9, 0x7ff80000
-; GFX11-NEXT:    s_and_b32 s8, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s8, s2, s6
-; GFX11-NEXT:    s_and_b32 s10, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s13, s[0:1], s[4:5]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, s[8:9], 0
-; GFX11-NEXT:    s_and_b32 s17, s11, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX11-NEXT:    s_and_b32 s17, s12, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s7, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX11-NEXT:    s_and_b32 s7, s11, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX11-NEXT:    s_and_b32 s7, s12, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX11-NEXT:    s_and_b32 s6, s10, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    s_and_b32 s6, s13, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s1, s5
-; GFX11-NEXT:    s_and_b32 s7, s14, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s7, s6, 0x7ff80000
-; GFX11-NEXT:    s_and_b32 s6, s13, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s0, s4
-; GFX11-NEXT:    s_and_b32 s8, s14, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s6, s6, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    s_and_b32 s9, s15, exec_lo
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, s[6:7], 0
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX11-NEXT:    s_and_b32 s9, s16, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s5, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX11-NEXT:    s_and_b32 s5, s15, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s6
-; GFX11-NEXT:    s_and_b32 s5, s16, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s4, s0
-; GFX11-NEXT:    s_and_b32 s4, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s6
+; GFX11-NEXT:    v_min_f64 v[0:1], s[2:3], s[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[6:7]
+; GFX11-NEXT:    v_min_f64 v[4:5], s[0:1], s[4:5]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[4:5]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v1, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, s0
 ; GFX11-NEXT:    ;;#ASMSTART
-; GFX11-NEXT:    ; use s[0:3]
+; GFX11-NEXT:    ; use v[0:3]
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1554,306 +979,110 @@ define <3 x double> @v_minimum_v3f64(<3 x double> %src0, <3 x double> %src1) {
 ; GFX7-LABEL: v_minimum_v3f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 32
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 32
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[8:9], 32
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[10:11]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v11, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, v8, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_mov_b32_e32 v14, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v14, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, 0, v12, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v11, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX940-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v14, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v10, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v12, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v12, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[8:9]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v12, vcc
+; GFX940-NEXT:    v_min_f64 v[6:7], v[4:5], v[10:11]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v12, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v10, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0x7ff80000, v12, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, 0x7ff80000, v14, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, v17, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, v18, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[8:9], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[6:7], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[12:13]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[14:15]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[16:17]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s11
+; GFX10-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX10-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v12, v7, v1 :: v_dual_cndmask_b32 v17, v6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v10, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, 0x7ff80000, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, 0x7ff80000, v14, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, 0, v17, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, 0, v18, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s4
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[8:9], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[10:11], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[12:13]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[14:15]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s7
+; GFX11-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX11-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[10:11]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f64:
@@ -1875,247 +1104,49 @@ define <3 x double> @v_minimum_v3f64__nnan(<3 x double> %src0, <3 x double> %src
 ; GFX7-LABEL: v_minimum_v3f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[6:7], 32
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v11, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v10, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[6:7]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[6:7], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[12:13]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 32
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[0:1]
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v11, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v10, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[6:7]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[8:9], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[6:7], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v10, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[12:13]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[14:15]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[16:17]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s11
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[8:9], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[10:11], 32
-; GFX11-NEXT:    v_dual_cndmask_b32 v13, v7, v1 :: v_dual_cndmask_b32 v12, v6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v10, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[12:13]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[14:15]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, v0, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v14, v2, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v15, v3, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s7
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f64__nnan:
@@ -2137,144 +1168,110 @@ define <3 x double> @v_minimum_v3f64__nsz(<3 x double> %src0, <3 x double> %src1
 ; GFX7-LABEL: v_minimum_v3f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX7-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX8-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v2, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[10:11]
+; GFX9-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11]
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v6, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v6, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v10, v4, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; GFX940-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[6:7]
+; GFX940-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v7, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v6, v5, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v12, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v13, v12, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[8:9]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v7, v12, vcc
+; GFX940-NEXT:    v_min_f64 v[6:7], v[4:5], v[10:11]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v6, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v7, v12, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v10, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s8
+; GFX10-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX10-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v12, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v10, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s4
+; GFX11-NEXT:    v_min_f64 v[12:13], v[0:1], v[6:7]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7]
+; GFX11-NEXT:    v_min_f64 v[6:7], v[2:3], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[8:9], v[4:5], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[10:11]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v12, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v6, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v8, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f64__nsz:
@@ -2296,88 +1293,49 @@ define <3 x double> @v_minimum_v3f64__nnan_nsz(<3 x double> %src0, <3 x double>
 ; GFX7-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[10:11]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s5
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[10:11]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s1
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[4:5], v[10:11]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f64__nnan_nsz:
@@ -2399,404 +1357,135 @@ define <4 x double> @v_minimum_v4f64(<4 x double> %src0, <4 x double> %src1) {
 ; GFX7-LABEL: v_minimum_v4f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX7-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX7-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX7-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX8-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX8-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX8-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s[6:7]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[10:11], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v18, v19, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v13, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v18, v10, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v15, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v18, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[12:13]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[12:13], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[14:15], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[10:11]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX9-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX9-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_mov_b32_e32 v18, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v18, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[12:13], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v15, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v18, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[14:15], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v16, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v16, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v16, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[4:5], v[12:13]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[12:13]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v16, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[6:7], v[14:15]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[14:15]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s6, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s7, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s8, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s10, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[14:15], 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v13, v5, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v15, v7, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, v19, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, 0x7ff80000, v18, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v12, v4, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, v21, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v14, v6, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[4:5], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[6:7], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0x7ff80000, v20, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, v23, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0x7ff80000, v22, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, v24, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s9, v[10:11], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s6, 0, v[16:17]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[18:19]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[20:21]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[22:23]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s14
+; GFX10-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[10:11]
+; GFX10-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[12:13]
+; GFX10-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s2, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s3, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s6, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v9, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v11, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v13, v5, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v15, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, 0x7ff80000, v16, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, 0x7ff80000, v18, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, 0x7ff80000, v20, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v14, v6, s3
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, 0x7ff80000, v22, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, 0, v20, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, 0, v24, s6
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[12:13], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[14:15], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[10:11], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[18:19]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[20:21]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[22:23]
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s8
+; GFX11-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[10:11]
+; GFX11-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[12:13]
+; GFX11-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[14:15]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f64:
@@ -2819,320 +1508,55 @@ define <4 x double> @v_minimum_v4f64__nnan(<4 x double> %src0, <4 x double> %src
 ; GFX7-LABEL: v_minimum_v4f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX7-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX8-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[0:1], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[8:9], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[10:11], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v10, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v13, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v12, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[10:11], v[14:15], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v15, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v14, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v6, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v7, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX9-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[8:9], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v11, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v10, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[10:11], 32
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v13, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v12, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[12:13], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[0:1]
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v9, v5, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v15, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v14, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[14:15], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[8:9]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX940-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s6, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[10:11], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[8:9], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[14:15], 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v15, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s5
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[4:5], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v14, v6, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[6:7], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[2:3], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[16:17]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[18:19]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[20:21]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[22:23]
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v20, v4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s14
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX10-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s2, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[6:7], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[14:15], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[10:11], 32
-; GFX11-NEXT:    v_dual_cndmask_b32 v17, v9, v1 :: v_dual_cndmask_b32 v16, v8, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v13, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v12, v4, s1
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[4:5], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, v15, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v14, v6, s2
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[12:13], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[16:17]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s8, 0, v[18:19]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[20:21]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[22:23]
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s4
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v16, v0 :: v_dual_cndmask_b32 v1, v17, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v20, v4, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v22, v6, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, v1, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v21, v5, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v23, v7, s10
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f64__nnan:
@@ -3155,180 +1579,135 @@ define <4 x double> @v_minimum_v4f64__nsz(<4 x double> %src0, <4 x double> %src1
 ; GFX7-LABEL: v_minimum_v4f64__nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX7-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX7-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f64__nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX8-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX8-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f64__nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v14, v6, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v12, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11]
+; GFX9-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13]
+; GFX9-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v7, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v11, v7, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v13, v7, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f64__nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v8, v1, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v10, v2, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v12, v4, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v14, v6, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v15, v7, vcc
+; GFX940-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, 0, v9, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v16, 0x7ff80000
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v16, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[10:11]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v16, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[4:5], v[12:13]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[12:13]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v16, vcc
+; GFX940-NEXT:    v_min_f64 v[8:9], v[6:7], v[14:15]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[14:15]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v8, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v16, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f64__nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s7, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s10, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v12, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v14, v6, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, v10, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0x7ff80000, v7, s10
+; GFX10-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[10:11]
+; GFX10-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[12:13]
+; GFX10-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f64__nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s2, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s3, v[6:7], v[14:15]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s4, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s6, v[6:7], v[14:15]
-; GFX11-NEXT:    v_dual_cndmask_b32 v16, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v10, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v12, v4, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v14, v6, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, v16, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, v10, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, 0, v12, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0x7ff80000, v1, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0x7ff80000, v3, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v5, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, 0x7ff80000, v7, s6
+; GFX11-NEXT:    v_min_f64 v[16:17], v[0:1], v[8:9]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[8:9], v[2:3], v[10:11]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[10:11]
+; GFX11-NEXT:    v_min_f64 v[10:11], v[4:5], v[12:13]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[12:13]
+; GFX11-NEXT:    v_min_f64 v[12:13], v[6:7], v[14:15]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[14:15]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v8, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v10, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v12, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f64__nsz:
@@ -3351,108 +1730,55 @@ define <4 x double> @v_minimum_v4f64__nnan_nsz(<4 x double> %src0, <4 x double>
 ; GFX7-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX7-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX8-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s[8:9]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX9-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v15, v7, vcc
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX940-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX940-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX940-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[2:3], v[10:11]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[4:5], v[12:13]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s6, v[6:7], v[14:15]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s6
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX10-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX10-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX10-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f64__nnan_nsz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[2:3], v[10:11]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[4:5], v[12:13]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s2, v[6:7], v[14:15]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v15, v7, s2
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-NEXT:    v_min_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-NEXT:    v_min_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-NEXT:    v_min_f64 v[6:7], v[6:7], v[14:15]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f64__nnan_nsz:
@@ -3475,782 +1801,244 @@ define <8 x double> @v_minimum_v8f64(<8 x double> %src0, <8 x double> %src1) {
 ; GFX7-LABEL: v_minimum_v8f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 32
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX7-NEXT:    v_min_f64 v[32:33], v[2:3], v[18:19]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX7-NEXT:    v_min_f64 v[18:19], v[4:5], v[20:21]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[0:1], v[16:17]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX7-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX7-NEXT:    v_min_f64 v[20:21], v[6:7], v[22:23]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX7-NEXT:    v_min_f64 v[16:17], v[8:9], v[24:25]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX7-NEXT:    v_min_f64 v[22:23], v[10:11], v[26:27]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX7-NEXT:    v_min_f64 v[24:25], v[12:13], v[28:29]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX7-NEXT:    v_min_f64 v[18:19], v[14:15], v[30:31]
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v8f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 32
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX8-NEXT:    v_min_f64 v[32:33], v[2:3], v[18:19]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX8-NEXT:    v_min_f64 v[18:19], v[4:5], v[20:21]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[0:1], v[16:17]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX8-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX8-NEXT:    v_min_f64 v[20:21], v[6:7], v[22:23]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX8-NEXT:    v_min_f64 v[16:17], v[8:9], v[24:25]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX8-NEXT:    v_min_f64 v[22:23], v[10:11], v[26:27]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX8-NEXT:    v_min_f64 v[24:25], v[12:13], v[28:29]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX8-NEXT:    v_min_f64 v[18:19], v[14:15], v[30:31]
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v8f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29]
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v17, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v32, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v16, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v33, 0, v31, s[4:5]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19]
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v33, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v34, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, v21, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v20, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v18, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[6:7]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[4:5]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v23, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[22:23], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v25, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v24, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[18:19]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[8:9], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[24:25], 32
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v19, v7, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[16:17]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[26:27], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v27, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v16, s[8:9]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[8:9], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[4:5]
+; GFX9-NEXT:    v_min_f64 v[32:33], v[2:3], v[18:19]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX9-NEXT:    v_min_f64 v[18:19], v[4:5], v[20:21]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[0:1], v[16:17]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17]
+; GFX9-NEXT:    v_mov_b32_e32 v34, 0x7ff80000
+; GFX9-NEXT:    v_min_f64 v[20:21], v[6:7], v[22:23]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23]
+; GFX9-NEXT:    v_min_f64 v[16:17], v[8:9], v[24:25]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25]
+; GFX9-NEXT:    v_min_f64 v[22:23], v[10:11], v[26:27]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27]
+; GFX9-NEXT:    v_min_f64 v[24:25], v[12:13], v[28:29]
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, v34, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v32, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v33, v34, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v19, v34, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v21, v34, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v16, 0, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v17, v34, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v22, 0, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v23, v34, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v24, 0, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v25, v34, s[14:15]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v29, v13, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v28, v12, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v18, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v19, v11, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v18, v10, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v19, v11, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[12:13], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[12:13]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[28:29], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v31, v15, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v32, v18, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v30, v14, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, v18, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[30:31], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[16:17]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[18:19]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v18, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v19, v15, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v18, v14, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v19, v15, s[12:13]
+; GFX9-NEXT:    v_min_f64 v[18:19], v[14:15], v[30:31]
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v18, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v19, v34, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v8f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17]
-; GFX940-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v17, v1, vcc
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v35, v32, v33, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v33, v16, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v34, 0, v33, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[16:17], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v34, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v35, v1, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v19, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[18:19]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v34, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v35, v1, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v18, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[18:19], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[20:21]
-; GFX940-NEXT:    v_cndmask_b32_e64 v2, v16, v2, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v21, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[20:21]
-; GFX940-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v20, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[20:21], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23]
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v16, v4, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v23, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[22:23]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v22, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[22:23], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[8:9], v[24:25]
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v16, v6, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v25, v9, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[8:9], v[24:25]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v24, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[24:25], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v16, v8, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v27, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[10:11], v[26:27]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v26, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[26:27], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[12:13], v[28:29]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v16, v10, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v29, v13, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[12:13], v[28:29]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v28, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[28:29], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
+; GFX940-NEXT:    v_mov_b32_e32 v54, 0x7ff80000
+; GFX940-NEXT:    v_min_f64 v[32:33], v[0:1], v[16:17]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[16:17]
+; GFX940-NEXT:    v_min_f64 v[34:35], v[2:3], v[18:19]
+; GFX940-NEXT:    v_min_f64 v[36:37], v[4:5], v[20:21]
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v33, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[18:19]
+; GFX940-NEXT:    v_min_f64 v[38:39], v[6:7], v[22:23]
+; GFX940-NEXT:    v_min_f64 v[48:49], v[8:9], v[24:25]
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v34, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v35, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[4:5], v[20:21]
+; GFX940-NEXT:    v_min_f64 v[50:51], v[10:11], v[26:27]
+; GFX940-NEXT:    v_min_f64 v[52:53], v[12:13], v[28:29]
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v36, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v37, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[6:7], v[22:23]
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v16, v12, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v31, v15, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[14:15], v[30:31]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v17, v13, s[2:3]
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v32, v16, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, 0, v16, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[30:31], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[16:17]
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v15, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v16, v14, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v17, v15, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[16:17], v[14:15], v[30:31]
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v38, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v39, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[8:9], v[24:25]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v8, v48, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v49, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[10:11], v[26:27]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v10, v50, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v51, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[12:13], v[28:29]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v12, v52, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v53, v54, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[30:31]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v14, v16, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v54, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v8f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[16:17]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[0:1], v[16:17]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s9, v[6:7], v[22:23]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s10, v[8:9], v[24:25]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s11, v[10:11], v[26:27]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s12, v[12:13], v[28:29]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s13, v[6:7], v[22:23]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s14, v[8:9], v[24:25]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s15, v[10:11], v[26:27]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s16, v[12:13], v[28:29]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[2:3], v[18:19]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[18:19]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s7, v[4:5], v[20:21]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[20:21]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[26:27], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s18, v[28:29], 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v17, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v23, v7, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v25, v9, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v27, v11, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v33, 0x7ff80000, v32, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, v29, v13, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v38, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v48, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, 0, v32, s4
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v50, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v52, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v22, v6, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v24, v8, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v26, v10, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, v28, v12, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, v[16:17], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[18:19], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v19, v3, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, v21, v5, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 s9, v[12:13], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v35, 0x7ff80000, v34, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v18, v2, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, v20, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[4:5], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[6:7], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[8:9], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[10:11], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s16
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[20:21], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s15, v[22:23], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s16, v[24:25], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[32:33]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[34:35]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[36:37]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[48:49]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[50:51]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[52:53]
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v33, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v22, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v23, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s24
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s24
+; GFX10-NEXT:    v_min_f64 v[32:33], v[0:1], v[16:17]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17]
+; GFX10-NEXT:    v_min_f64 v[16:17], v[2:3], v[18:19]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[18:19]
+; GFX10-NEXT:    v_min_f64 v[18:19], v[4:5], v[20:21]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[20:21]
+; GFX10-NEXT:    v_min_f64 v[20:21], v[6:7], v[22:23]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[6:7], v[22:23]
+; GFX10-NEXT:    v_min_f64 v[22:23], v[8:9], v[24:25]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s7, v[8:9], v[24:25]
+; GFX10-NEXT:    v_min_f64 v[24:25], v[10:11], v[26:27]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s8, v[10:11], v[26:27]
+; GFX10-NEXT:    v_min_f64 v[26:27], v[12:13], v[28:29]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s9, v[12:13], v[28:29]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v16, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v17, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v19, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v21, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v22, 0, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v23, 0x7ff80000, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v24, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v25, 0x7ff80000, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v26, 0, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v27, 0x7ff80000, s9
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s10, v[14:15], v[30:31]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s13, v[14:15], v[30:31]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s25, v[30:31], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v31, v15, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v30, v14, s10
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v16, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, 0, v18, s13
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[14:15], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e32 vcc_lo, 0, v[54:55]
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s25
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s25
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v54, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v55, v15, vcc_lo
+; GFX10-NEXT:    v_min_f64 v[28:29], v[14:15], v[30:31]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s10, v[14:15], v[30:31]
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v28, 0, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v29, 0x7ff80000, s10
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v8f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s4, v[6:7], v[22:23]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s9, v[6:7], v[22:23]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s1, v[2:3], v[18:19]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s6, v[10:11], v[26:27]
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[16:17]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s2, v[2:3], v[18:19]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s11, v[10:11], v[26:27]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[0:1], v[16:17]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s3, v[4:5], v[20:21]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s5, v[8:9], v[24:25]
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s7, v[12:13], v[28:29]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[20:21]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s10, v[8:9], v[24:25]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s12, v[12:13], v[28:29]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s13, v[18:19], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s15, v[20:21], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, v23, v7, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, v19, v3, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, v27, v11, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v38, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, v22, v6, s4
-; GFX11-NEXT:    v_cmp_class_f64_e64 s4, v[6:7], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v35, 0x7ff80000, v34, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v50, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, v18, v2, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, v26, v10, s6
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[0:1], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, v21, v5, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, v25, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, v29, v13, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s11
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, v[16:17], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v48, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v52, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, v20, v4, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, v24, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, v28, v12, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s9
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[2:3], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[4:5], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s5, v[8:9], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s7, v[10:11], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s9, v[12:13], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s12
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[24:25], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, v[26:27], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[28:29], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[34:35]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s16, 0, v[36:37]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[38:39]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s18, 0, v[48:49]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[50:51]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[52:53]
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s4
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v17, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v33, 0x7ff80000, v32, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v32, v16, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v32, 0, v32, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s1
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[32:33]
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v20, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v24, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v26, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v28, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v21, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v25, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v27, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v29, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v34, v2, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v36, v4, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v48, v8, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v50, v10, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v52, v12, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v35, v3, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v37, v5, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v49, v9, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v51, v11, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v53, v13, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, v0, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, v1, s12
+; GFX11-NEXT:    v_min_f64 v[32:33], v[0:1], v[16:17]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17]
+; GFX11-NEXT:    v_min_f64 v[16:17], v[2:3], v[18:19]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[18:19]
+; GFX11-NEXT:    v_min_f64 v[18:19], v[4:5], v[20:21]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[20:21]
+; GFX11-NEXT:    v_min_f64 v[20:21], v[6:7], v[22:23]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[22:23]
+; GFX11-NEXT:    v_min_f64 v[22:23], v[8:9], v[24:25]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s3, v[8:9], v[24:25]
+; GFX11-NEXT:    v_min_f64 v[24:25], v[10:11], v[26:27]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s4, v[10:11], v[26:27]
+; GFX11-NEXT:    v_min_f64 v[26:27], v[12:13], v[28:29]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s5, v[12:13], v[28:29]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v32, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v16, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v17, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v18, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v19, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v20, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v21, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v8, v22, 0, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v23, 0x7ff80000, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v24, 0, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v25, 0x7ff80000, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v26, 0, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v27, 0x7ff80000, s5
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[14:15], v[30:31]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s0, v[14:15], v[30:31]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s19, v[30:31], 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v54, v31, v15, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v30, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v54, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v54, 0, v16, s0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[14:15], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[54:55]
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_cndmask_b32 v6, v6, v22
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v38, v6, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v39, v7, s17
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v30, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v31, s19
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v54, v14, s22
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v55, v15, s22
+; GFX11-NEXT:    v_min_f64 v[28:29], v[14:15], v[30:31]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s6, v[14:15], v[30:31]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v28, 0, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v29, 0x7ff80000, s6
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v8f64:
@@ -4279,1799 +2067,798 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-LABEL: v_minimum_v16f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX7-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX7-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX7-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX7-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX7-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX7-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX7-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX7-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX7-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX7-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[2:3], v[2:3], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[4:5], v[4:5], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[6:7], v[6:7], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[8:9], v[8:9], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[10:11], v[10:11], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 32
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX7-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX7-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX7-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[12:13], v[12:13], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX7-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[14:15], v[14:15], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[16:17], v[16:17], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX7-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[18:19], v[18:19], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX7-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX7-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 32
-; GFX7-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX7-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX7-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX7-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX7-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX7-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX7-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX7-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[20:21], v[20:21], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX7-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[22:23], v[22:23], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX7-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[24:25], v[24:25], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX7-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[26:27], v[26:27], v[31:32]
+; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX7-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 32
-; GFX7-NEXT:    s_waitcnt vmcnt(7)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(5)
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX7-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX7-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX7-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX7-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
+; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX7-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX7-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX7-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX7-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX7-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX7-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX7-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX7-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX7-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX7-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 32
-; GFX7-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v16f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX8-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX8-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX8-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX8-NEXT:    s_waitcnt vmcnt(2)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[2:3], v[2:3], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[4:5], v[4:5], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX8-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX8-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[6:7], v[6:7], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[8:9], v[8:9], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[10:11], v[10:11], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 32
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX8-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX8-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX8-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[12:13], v[12:13], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[14:15], v[14:15], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[16:17], v[16:17], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[18:19], v[18:19], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX8-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX8-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 32
-; GFX8-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX8-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX8-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX8-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX8-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[20:21], v[20:21], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX8-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[22:23], v[22:23], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[24:25], v[24:25], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX8-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[26:27], v[26:27], v[31:32]
+; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX8-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 32
-; GFX8-NEXT:    s_waitcnt vmcnt(7)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(5)
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX8-NEXT:    s_waitcnt vmcnt(3)
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX8-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX8-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
+; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX8-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX8-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX8-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX8-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX8-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX8-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX8-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX8-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX8-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 32
-; GFX8-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v16f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    v_mov_b32_e32 v39, 0x7ff80000
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v48, v32, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v48, v31, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v32, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v34, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v33, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v32, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, 0, v31, s[6:7]
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v34, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[35:36], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v36, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v33, v35, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v33, 0, v33, s[6:7]
+; GFX9-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX9-NEXT:    v_writelane_b32 v34, s35, 3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v35, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v36, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v38, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v35, v37, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v38, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v48, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v49, v1, vcc
-; GFX9-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v31, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v32, v3, vcc
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[4:5], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[6:7]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[6:7], v[6:7], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s[8:9]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[8:9], v[8:9], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, 0, s[10:11]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[10:11], v[10:11], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v33, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v34, v5, vcc
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v35, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v36, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 32
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v38, v9, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v37, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v35, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v36, v9, vcc
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v38, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v35, v8, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v49, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v35, v48, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v35, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v36, v9, s[6:7]
-; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v49, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v50, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v32, v13, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v51, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v31, v12, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[31:32], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v31, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, 0, s[12:13]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[12:13], v[12:13], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, 0, s[14:15]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[14:15], v[14:15], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, 0, s[16:17]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[16:17], v[16:17], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, 0, s[18:19]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[18:19], v[18:19], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:88
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v48, v12, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v49, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v34, v15, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v33, v14, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v34, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v48, v14, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v49, v15, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, v38, v17, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v37, v16, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v49, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[37:38], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, v37, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v17, v38, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v50, v36, v19, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v37, v35, v18, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[48:49]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[18:19], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v51, v39, v50, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v50, 0, v37, s[6:7]
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v48, v16, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v17, v49, v17, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[35:36], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, v50, v18, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v51, v19, s[4:5]
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v18, v35, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v36, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e64 v48, v32, v21, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v31, v20, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[20:21], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v48, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v50, v18, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v51, v19, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[31:32], 32
-; GFX9-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[35:36]
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34]
-; GFX9-NEXT:    v_cndmask_b32_e32 v20, v20, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v21, v21, v32, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v35, v20, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v21, v36, v21, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[33:34], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v34, v23, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v39, v31, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v33, v22, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, 0, v31, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[35:36]
-; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, v33, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, v18, 0, s[20:21]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[20:21], v[20:21], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    v_cndmask_b32_e64 v20, v20, 0, s[22:23]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[22:23], v[22:23], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[24:25]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[24:25], v[24:25], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    v_cndmask_b32_e64 v24, v24, 0, s[26:27]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[26:27], v[26:27], v[31:32]
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX9-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX9-NEXT:    v_cndmask_b32_e64 v23, v23, v34, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 32
-; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v38, v25, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v34, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v37, v24, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v24, v34, v24, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v25, v35, v25, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[37:38], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[4:5], 0, v[34:35]
-; GFX9-NEXT:    v_cndmask_b32_e32 v24, v24, v37, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v25, v25, v38, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v24, v34, v24, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v25, v35, v25, s[4:5]
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[48:49], 32
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v49, v27, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v48, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[8:9]
-; GFX9-NEXT:    v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51]
-; GFX9-NEXT:    v_cndmask_b32_e32 v26, v34, v26, vcc
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[34:35]
-; GFX9-NEXT:    v_cndmask_b32_e32 v27, v35, v27, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, v48, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v27, v27, v49, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, v51, v29, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v27, v35, v27, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX9-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
+; GFX9-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v32, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v32, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v32, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v32, s[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v32, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v32, s[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v15, v15, v32, s[16:17]
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, v17, v32, s[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, v19, v32, s[20:21]
+; GFX9-NEXT:    v_cndmask_b32_e64 v21, v21, v32, s[22:23]
+; GFX9-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
+; GFX9-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
+; GFX9-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
+; GFX9-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s35, v34, 3
+; GFX9-NEXT:    v_readlane_b32 s34, v34, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v34, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v34, 0
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX9-NEXT:    v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33]
-; GFX9-NEXT:    v_cndmask_b32_e64 v35, v39, v36, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v26, v34, v26, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, v50, v28, s[8:9]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[6:7], v[28:29], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[10:11]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], v[50:51], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v36, v33, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v37, v39, v36, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v36, v32, v30, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 32
-; GFX9-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[4:5]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[32:33], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[10:11], 0, v[34:35]
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[12:13], 0, v[36:37]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, v50, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e32 v30, v36, v30, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v29, v51, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v34, v28, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v35, v29, s[10:11]
-; GFX9-NEXT:    v_cndmask_b32_e64 v30, v36, v30, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v37, v31, s[12:13]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v16f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:8
-; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:4
-; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:16
-; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:12
-; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:24
-; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:20
-; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:32
-; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:28
+; GFX940-NEXT:    v_accvgpr_write_b32 a1, v40 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a2, v41 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a3, v42 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a4, v43 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a5, v44 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a6, v45 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a7, v46 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a8, v47 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a9, v56 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a10, v57 ; Reload Reuse
+; GFX940-NEXT:    scratch_load_dword v37, off, s32 offset:16
+; GFX940-NEXT:    scratch_load_dword v36, off, s32 offset:12
+; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:24
+; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:20
+; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:32
+; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:28
+; GFX940-NEXT:    scratch_load_dword v57, off, s32 offset:8
+; GFX940-NEXT:    scratch_load_dword v56, off, s32 offset:4
+; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:40
+; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:36
+; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:48
+; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:44
+; GFX940-NEXT:    scratch_load_dword v43, off, s32 offset:56
+; GFX940-NEXT:    scratch_load_dword v42, off, s32 offset:52
+; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:64
+; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:60
+; GFX940-NEXT:    scratch_load_dword v55, off, s32 offset:72
+; GFX940-NEXT:    scratch_load_dword v54, off, s32 offset:68
+; GFX940-NEXT:    scratch_load_dword v53, off, s32 offset:80
+; GFX940-NEXT:    scratch_load_dword v52, off, s32 offset:76
+; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:88
+; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:84
+; GFX940-NEXT:    scratch_load_dword v35, off, s32 offset:96
+; GFX940-NEXT:    scratch_load_dword v34, off, s32 offset:92
 ; GFX940-NEXT:    scratch_load_dword v31, off, s32
-; GFX940-NEXT:    scratch_load_dword v33, off, s32 offset:128
-; GFX940-NEXT:    scratch_load_dword v32, off, s32 offset:124
-; GFX940-NEXT:    scratch_load_dword v35, off, s32 offset:120
-; GFX940-NEXT:    scratch_load_dword v34, off, s32 offset:116
-; GFX940-NEXT:    scratch_load_dword v43, off, s32 offset:40
-; GFX940-NEXT:    scratch_load_dword v42, off, s32 offset:36
+; GFX940-NEXT:    scratch_load_dword v33, off, s32 offset:104
+; GFX940-NEXT:    scratch_load_dword v32, off, s32 offset:100
+; GFX940-NEXT:    v_accvgpr_write_b32 a11, v58 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a12, v59 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a13, v60 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a14, v61 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a15, v62 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_write_b32 a16, v63 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_min_f64 v[58:59], v[2:3], v[36:37]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[2:3], v[36:37]
 ; GFX940-NEXT:    scratch_load_dword v37, off, s32 offset:112
 ; GFX940-NEXT:    scratch_load_dword v36, off, s32 offset:108
-; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:104
-; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:100
-; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:96
-; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:92
-; GFX940-NEXT:    scratch_load_dword v53, off, s32 offset:56
-; GFX940-NEXT:    scratch_load_dword v52, off, s32 offset:52
-; GFX940-NEXT:    scratch_load_dword v55, off, s32 offset:48
-; GFX940-NEXT:    scratch_load_dword v54, off, s32 offset:44
-; GFX940-NEXT:    v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX940-NEXT:    v_mov_b32_e32 v56, 0x7ff80000
-; GFX940-NEXT:    v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_write_b32 a13, v61 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_min_f64 v[60:61], v[4:5], v[38:39]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[0:1], v[4:5], v[38:39]
+; GFX940-NEXT:    scratch_load_dword v39, off, s32 offset:120
+; GFX940-NEXT:    scratch_load_dword v38, off, s32 offset:116
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_min_f64 v[62:63], v[6:7], v[48:49]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[2:3], v[6:7], v[48:49]
+; GFX940-NEXT:    scratch_load_dword v49, off, s32 offset:128
+; GFX940-NEXT:    scratch_load_dword v48, off, s32 offset:124
+; GFX940-NEXT:    s_waitcnt vmcnt(25)
+; GFX940-NEXT:    v_min_f64 v[2:3], v[0:1], v[56:57]
+; GFX940-NEXT:    v_cmp_u_f64_e64 s[4:5], v[0:1], v[56:57]
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x7ff80000
 ; GFX940-NEXT:    s_waitcnt vmcnt(23)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[40:41]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v41, v1, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[0:1], v[40:41]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v40, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v57, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[40:41], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v58, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v59, v1, vcc
+; GFX940-NEXT:    v_min_f64 v[56:57], v[8:9], v[46:47]
+; GFX940-NEXT:    v_cndmask_b32_e64 v1, v2, 0, s[4:5]
+; GFX940-NEXT:    v_accvgpr_write_b32 a0, v1
+; GFX940-NEXT:    v_cndmask_b32_e64 v1, v3, v0, s[4:5]
+; GFX940-NEXT:    v_cndmask_b32_e64 v2, v58, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v59, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[8:9], v[46:47]
 ; GFX940-NEXT:    s_waitcnt vmcnt(21)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[2:3], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v41, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v51, v3, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[2:3], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e32 v40, v50, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[2:3], 32
-; GFX940-NEXT:    v_cndmask_b32_e64 v61, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v60, 0, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v60, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v61, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[50:51], 32
-; GFX940-NEXT:    scratch_load_dword v41, off, s32 offset:64
-; GFX940-NEXT:    scratch_load_dword v40, off, s32 offset:60
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v50, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v51, vcc
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(21)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[4:5], v[44:45]
-; GFX940-NEXT:    scratch_load_dword v51, off, s32 offset:88
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v58, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v59, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[60:61]
+; GFX940-NEXT:    v_min_f64 v[46:47], v[10:11], v[44:45]
+; GFX940-NEXT:    v_cndmask_b32_e64 v4, v60, 0, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v8, v56, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v57, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[10:11], v[44:45]
+; GFX940-NEXT:    s_waitcnt vmcnt(19)
+; GFX940-NEXT:    v_min_f64 v[44:45], v[12:13], v[42:43]
+; GFX940-NEXT:    v_cndmask_b32_e64 v5, v61, v0, s[0:1]
+; GFX940-NEXT:    v_cndmask_b32_e64 v10, v46, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v11, v47, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[12:13], v[42:43]
+; GFX940-NEXT:    s_waitcnt vmcnt(17)
+; GFX940-NEXT:    v_min_f64 v[42:43], v[14:15], v[40:41]
+; GFX940-NEXT:    v_cndmask_b32_e64 v6, v62, 0, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v12, v44, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v13, v45, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[14:15], v[40:41]
+; GFX940-NEXT:    s_waitcnt vmcnt(15)
+; GFX940-NEXT:    v_min_f64 v[40:41], v[16:17], v[54:55]
+; GFX940-NEXT:    v_cndmask_b32_e64 v7, v63, v0, s[2:3]
+; GFX940-NEXT:    v_cndmask_b32_e64 v14, v42, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v43, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[16:17], v[54:55]
+; GFX940-NEXT:    s_waitcnt vmcnt(13)
+; GFX940-NEXT:    v_min_f64 v[54:55], v[18:19], v[52:53]
+; GFX940-NEXT:    v_accvgpr_read_b32 v63, a16 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v16, v40, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v17, v41, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[18:19], v[52:53]
+; GFX940-NEXT:    s_waitcnt vmcnt(11)
+; GFX940-NEXT:    v_min_f64 v[52:53], v[20:21], v[50:51]
+; GFX940-NEXT:    v_accvgpr_read_b32 v62, a15 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v18, v54, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v19, v55, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[20:21], v[50:51]
+; GFX940-NEXT:    s_waitcnt vmcnt(9)
+; GFX940-NEXT:    v_min_f64 v[50:51], v[22:23], v[34:35]
+; GFX940-NEXT:    v_accvgpr_read_b32 v61, a14 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v20, v52, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v21, v53, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[22:23], v[34:35]
+; GFX940-NEXT:    s_waitcnt vmcnt(6)
+; GFX940-NEXT:    v_min_f64 v[34:35], v[24:25], v[32:33]
+; GFX940-NEXT:    v_accvgpr_read_b32 v60, a13 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v22, v50, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v23, v51, v0, vcc
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[24:25], v[32:33]
+; GFX940-NEXT:    v_accvgpr_read_b32 v59, a12 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v58, a11 ; Reload Reuse
+; GFX940-NEXT:    v_cndmask_b32_e64 v24, v34, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v25, v35, v0, vcc
+; GFX940-NEXT:    v_accvgpr_read_b32 v57, a10 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v56, a9 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v47, a8 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v46, a7 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v45, a6 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v44, a5 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v43, a4 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v42, a3 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v41, a2 ; Reload Reuse
+; GFX940-NEXT:    v_accvgpr_read_b32 v40, a1 ; Reload Reuse
+; GFX940-NEXT:    s_waitcnt vmcnt(4)
+; GFX940-NEXT:    v_min_f64 v[32:33], v[26:27], v[36:37]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[26:27], v[36:37]
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v60, v2, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v61, v3, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[4:5], v[44:45]
-; GFX940-NEXT:    v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v60, a12 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v45, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v44, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[4:5], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[44:45], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v58, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v59, v5, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, v44, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v5, v5, v45, s[0:1]
-; GFX940-NEXT:    scratch_load_dword v45, off, s32 offset:72
-; GFX940-NEXT:    scratch_load_dword v44, off, s32 offset:68
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(22)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[6:7], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v58, v4, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v59, v5, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[6:7], v[46:47]
+; GFX940-NEXT:    v_cndmask_b32_e64 v26, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v27, v33, v0, vcc
+; GFX940-NEXT:    s_waitcnt vmcnt(2)
+; GFX940-NEXT:    v_min_f64 v[32:33], v[28:29], v[38:39]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[28:29], v[38:39]
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v47, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v46, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[6:7], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[46:47], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v58, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v59, v7, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v6, v6, v46, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v7, v7, v47, s[0:1]
-; GFX940-NEXT:    scratch_load_dword v47, off, s32 offset:80
-; GFX940-NEXT:    scratch_load_dword v46, off, s32 offset:76
-; GFX940-NEXT:    scratch_load_dword v50, off, s32 offset:84
-; GFX940-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[58:59]
-; GFX940-NEXT:    s_waitcnt vmcnt(18)
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[8:9], v[42:43]
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v58, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v59, v7, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[8:9], v[42:43]
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v43, v9, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v59, v56, v57, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v57, v42, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v58, 0, v57, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[8:9], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[42:43], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[58:59]
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v58, v8, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v59, v9, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(8)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[10:11], v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v8, v42, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v9, v43, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v42, v55, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[10:11], v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e64 v8, v58, v8, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v9, v59, v9, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v43, v56, v42, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v42, v54, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v42, 0, v42, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[10:11], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[54:55], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[42:43]
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v42, v10, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v43, v11, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[12:13], v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v10, v54, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v11, v55, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v54, v53, v13, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[12:13], v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e64 v10, v42, v10, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v11, v43, v11, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v55, v56, v54, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v54, v52, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v54, 0, v54, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[12:13], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[52:53], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[54:55]
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(6)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[14:15], v[40:41]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v12, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v13, v53, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v41, v15, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[14:15], v[40:41]
-; GFX940-NEXT:    v_cndmask_b32_e64 v12, v54, v12, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v13, v55, v13, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v40, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[14:15], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[40:41], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v52, v14, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v14, v40, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v53, v15, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(3)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[16:17], v[44:45]
-; GFX940-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v15, v41, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v45, v17, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[16:17], v[44:45]
-; GFX940-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v44, v16, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[16:17], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[44:45], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v52, v16, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, v16, v44, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v17, v53, v17, vcc
-; GFX940-NEXT:    s_waitcnt vmcnt(1)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[18:19], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e64 v16, v52, v16, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v17, v45, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v47, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[18:19], v[46:47]
-; GFX940-NEXT:    v_cndmask_b32_e64 v17, v53, v17, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v46, v18, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[18:19], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[46:47], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v52, v18, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v18, v18, v46, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v53, v19, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v28, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v29, v33, v0, vcc
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[20:21], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v18, v52, v18, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v19, v19, v47, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v51, v21, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[20:21], v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e64 v19, v53, v19, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX940-NEXT:    v_cndmask_b32_e64 v53, v56, v52, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v52, v50, v20, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v52, 0, v52, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[20:21], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[50:51], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[52:53]
-; GFX940-NEXT:    v_cndmask_b32_e32 v20, v52, v20, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v21, v53, v21, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[22:23], v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e64 v20, v20, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v21, v21, v51, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v49, v23, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[22:23], v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e64 v20, v52, v20, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v21, v53, v21, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v51, v56, v50, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v50, v48, v22, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v50, 0, v50, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[22:23], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[48:49], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[50:51]
-; GFX940-NEXT:    v_cndmask_b32_e32 v22, v50, v22, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v23, v51, v23, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[24:25], v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e64 v22, v22, v48, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v23, v23, v49, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v48, v39, v25, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[24:25], v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e64 v22, v50, v22, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v23, v51, v23, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v49, v56, v48, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v48, v38, v24, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v48, 0, v48, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[24:25], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[38:39], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[48:49]
-; GFX940-NEXT:    v_cndmask_b32_e32 v24, v48, v24, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v25, v49, v25, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[26:27], v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e64 v24, v24, v38, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v25, v25, v39, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v38, v37, v27, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[26:27], v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e64 v24, v48, v24, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v25, v49, v25, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v39, v56, v38, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v38, v36, v26, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v38, 0, v38, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[26:27], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[36:37], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[38:39]
-; GFX940-NEXT:    v_cndmask_b32_e32 v26, v38, v26, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v27, v39, v27, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[28:29], v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e64 v26, v26, v36, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v27, v27, v37, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v36, v35, v29, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[28:29], v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e64 v26, v38, v26, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v27, v39, v27, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v37, v56, v36, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v36, v34, v28, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v36, 0, v36, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[28:29], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[34:35], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[36:37]
-; GFX940-NEXT:    v_cndmask_b32_e32 v28, v36, v28, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v29, v37, v29, vcc
-; GFX940-NEXT:    v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33]
-; GFX940-NEXT:    v_cndmask_b32_e64 v28, v28, v34, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v29, v29, v35, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v34, v33, v31, vcc
-; GFX940-NEXT:    v_cmp_o_f64_e64 s[0:1], v[30:31], v[32:33]
-; GFX940-NEXT:    v_cndmask_b32_e64 v28, v36, v28, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v29, v37, v29, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v35, v56, v34, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e32 v34, v32, v30, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v34, 0, v34, s[0:1]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[30:31], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[32:33], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[34:35]
-; GFX940-NEXT:    v_cndmask_b32_e32 v30, v34, v30, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v31, v35, v31, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v30, v30, v32, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v31, v31, v33, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v30, v34, v30, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v31, v35, v31, s[2:3]
-; GFX940-NEXT:    v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX940-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX940-NEXT:    v_min_f64 v[32:33], v[30:31], v[48:49]
+; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[30:31], v[48:49]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e64 v30, v32, 0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v31, v33, v0, vcc
+; GFX940-NEXT:    v_accvgpr_read_b32 v0, a0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v16f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x20
+; GFX10-NEXT:    s_clause 0x19
+; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:32
+; GFX10-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:28
+; GFX10-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
+; GFX10-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:68
+; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; GFX10-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
+; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:56
+; GFX10-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:52
+; GFX10-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:48
+; GFX10-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:44
+; GFX10-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:40
 ; GFX10-NEXT:    buffer_load_dword v65, off, s[0:3], s32 offset:8
 ; GFX10-NEXT:    buffer_load_dword v64, off, s[0:3], s32 offset:4
-; GFX10-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:16
-; GFX10-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:12
-; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:24
-; GFX10-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:20
-; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:32
-; GFX10-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:28
-; GFX10-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:36
-; GFX10-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:40
-; GFX10-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:56
-; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:52
-; GFX10-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
-; GFX10-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:44
-; GFX10-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:60
-; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:72
-; GFX10-NEXT:    buffer_load_dword v83, off, s[0:3], s32 offset:80
-; GFX10-NEXT:    buffer_load_dword v82, off, s[0:3], s32 offset:76
-; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:88
-; GFX10-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:84
+; GFX10-NEXT:    buffer_load_dword v66, off, s[0:3], s32 offset:100
 ; GFX10-NEXT:    buffer_load_dword v69, off, s[0:3], s32 offset:96
 ; GFX10-NEXT:    buffer_load_dword v68, off, s[0:3], s32 offset:92
-; GFX10-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:100
-; GFX10-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:104
-; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:112
-; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:108
-; GFX10-NEXT:    buffer_load_dword v85, off, s[0:3], s32 offset:120
-; GFX10-NEXT:    buffer_load_dword v84, off, s[0:3], s32 offset:116
+; GFX10-NEXT:    buffer_load_dword v71, off, s[0:3], s32 offset:88
+; GFX10-NEXT:    buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:80
+; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:76
+; GFX10-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:72
+; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    v_min_f64 v[82:83], v[2:3], v[31:32]
+; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[2:3], v[31:32]
+; GFX10-NEXT:    s_waitcnt vmcnt(22)
+; GFX10-NEXT:    v_min_f64 v[84:85], v[4:5], v[33:34]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[4:5], v[33:34]
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:120
+; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:116
+; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:112
+; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:108
+; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    v_min_f64 v[32:33], v[6:7], v[35:36]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[6:7], v[35:36]
+; GFX10-NEXT:    s_clause 0x2
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT:    buffer_load_dword v87, off, s[0:3], s32 offset:128
-; GFX10-NEXT:    buffer_load_dword v86, off, s[0:3], s32 offset:124
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[2:3], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[10:11], 32
-; GFX10-NEXT:    s_waitcnt vmcnt(31)
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[64:65]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[0:1], v[64:65]
-; GFX10-NEXT:    s_waitcnt vmcnt(29)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s5, v[2:3], v[54:55]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[2:3], v[54:55]
-; GFX10-NEXT:    s_waitcnt vmcnt(27)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s7, v[4:5], v[52:53]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[4:5], v[52:53]
-; GFX10-NEXT:    s_waitcnt vmcnt(25)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s9, v[6:7], v[50:51]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[6:7], v[50:51]
+; GFX10-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:128
+; GFX10-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:124
 ; GFX10-NEXT:    s_waitcnt vmcnt(23)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s13, v[8:9], v[48:49]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[64:65], 32
+; GFX10-NEXT:    v_cmp_u_f64_e64 s10, v[14:15], v[50:51]
 ; GFX10-NEXT:    s_waitcnt vmcnt(21)
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s15, v[12:13], v[36:37]
-; GFX10-NEXT:    s_waitcnt vmcnt(17)
-; GFX10-NEXT:    v_cmp_o_f64_e64 s16, v[14:15], v[34:35]
-; GFX10-NEXT:    v_cndmask_b32_e32 v96, v64, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v97, v54, v2, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v99, v55, v3, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v100, v52, v4, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v96, 0, v96, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v101, v50, v6, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v98, 0, v97, s6
-; GFX10-NEXT:    v_cndmask_b32_e32 v97, v65, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[54:55], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v99, 0x7ff80000, v99, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v97, 0x7ff80000, v97, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v100, 0, v100, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v102, 0, v101, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[6:7], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[4:5], 32
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[10:11], v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v48, v8, s13
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[12:13], v[36:37]
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s6, v[14:15], v[34:35]
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v64, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v65, s14
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[52:53], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v113, v36, v12, s15
+; GFX10-NEXT:    v_cmp_u_f64_e64 s9, v[12:13], v[52:53]
+; GFX10-NEXT:    s_waitcnt vmcnt(19)
+; GFX10-NEXT:    v_cmp_u_f64_e64 s7, v[10:11], v[54:55]
+; GFX10-NEXT:    s_waitcnt vmcnt(18)
+; GFX10-NEXT:    v_min_f64 v[34:35], v[8:9], v[37:38]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s6, v[8:9], v[37:38]
+; GFX10-NEXT:    s_waitcnt vmcnt(16)
+; GFX10-NEXT:    v_min_f64 v[8:9], v[0:1], v[64:65]
+; GFX10-NEXT:    v_min_f64 v[36:37], v[10:11], v[54:55]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s8, v[0:1], v[64:65]
+; GFX10-NEXT:    v_min_f64 v[38:39], v[12:13], v[52:53]
+; GFX10-NEXT:    v_min_f64 v[52:53], v[14:15], v[50:51]
+; GFX10-NEXT:    s_waitcnt vmcnt(11)
+; GFX10-NEXT:    v_min_f64 v[54:55], v[20:21], v[70:71]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s13, v[20:21], v[70:71]
+; GFX10-NEXT:    s_waitcnt vmcnt(9)
+; GFX10-NEXT:    v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
+; GFX10-NEXT:    s_waitcnt vmcnt(8)
+; GFX10-NEXT:    v_min_f64 v[50:51], v[16:17], v[48:49]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s11, v[16:17], v[48:49]
+; GFX10-NEXT:    v_min_f64 v[48:49], v[18:19], v[80:81]
+; GFX10-NEXT:    v_min_f64 v[64:65], v[22:23], v[68:69]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
+; GFX10-NEXT:    s_waitcnt vmcnt(7)
+; GFX10-NEXT:    v_min_f64 v[68:69], v[24:25], v[66:67]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v36, 0, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0x7ff80000, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v34, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v35, 0x7ff80000, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v37, 0x7ff80000, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v38, 0, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v39, 0x7ff80000, s9
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, 0, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, 0x7ff80000, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, v50, 0, s11
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, v51, 0x7ff80000, s11
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, v48, 0, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v49, 0x7ff80000, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v20, v54, 0, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v21, v55, 0x7ff80000, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v22, v64, 0, s14
+; GFX10-NEXT:    v_cndmask_b32_e64 v23, v65, 0x7ff80000, s14
+; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
+; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
+; GFX10-NEXT:    s_waitcnt vmcnt(5)
+; GFX10-NEXT:    v_min_f64 v[70:71], v[28:29], v[2:3]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(3)
+; GFX10-NEXT:    v_min_f64 v[66:67], v[26:27], v[4:5]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v82, 0, vcc_lo
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_o_f64_e64 s18, v[30:31], v[86:87]
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v54, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, v53, v5, s7
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[50:51], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, v51, v7, s9
-; GFX10-NEXT:    v_cmp_o_f64_e64 s9, v[8:9], v[48:49]
-; GFX10-NEXT:    v_cndmask_b32_e64 v101, 0x7ff80000, v54, s8
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s7, v[16:17], v[32:33]
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v103, 0x7ff80000, v55, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[8:9], 32
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[10:11], v[38:39]
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[48:49], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v38, v10, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, v34, v14, s6
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[16:17], v[32:33]
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v52, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v53, s14
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s14, v[18:19], v[82:83]
-; GFX10-NEXT:    v_cndmask_b32_e64 v52, 0, v115, s16
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v50, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, v49, v9, s13
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[38:39], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v54, 0, v112, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, v39, v11, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v55, 0x7ff80000, v50, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v50, 0, v113, s5
-; GFX10-NEXT:    v_cmp_o_f64_e64 s4, v[18:19], v[82:83]
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v54, v8, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v64, 0, v114, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v55, v9, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[12:13], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v65, 0x7ff80000, v51, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v48, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v64, v10, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v49, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[14:15], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v65, v11, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, v37, v13, s15
-; GFX10-NEXT:    v_cmp_class_f64_e64 s17, v[34:35], 32
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s9, v[20:21], v[66:67]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s11, v[20:21], v[66:67]
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, v32, v16, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v51, 0x7ff80000, v48, s5
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v38, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f64_e32 vcc_lo, v[22:23], v[68:69]
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, v35, v15, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, v82, v18, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v48, 0, v116, s8
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[36:37], 32
-; GFX10-NEXT:    v_cmp_o_f64_e64 s5, v[22:23], v[68:69]
-; GFX10-NEXT:    v_cndmask_b32_e64 v53, 0x7ff80000, v38, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v50, v12, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v51, v13, s10
-; GFX10-NEXT:    v_cmp_class_f64_e64 s10, v[16:17], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v38, 0, v49, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v83, v19, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s12
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, v[32:33], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s14, v[18:19], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v67, v21, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v34, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v35, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, v33, v17, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v39, 0x7ff80000, v112, s4
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s4, v[24:25], v[70:71]
-; GFX10-NEXT:    v_cndmask_b32_e32 v113, v69, v23, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v68, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[20:21], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v49, 0x7ff80000, v34, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v34, 0x7ff80000, v114, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v36, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v37, s13
-; GFX10-NEXT:    v_cmp_class_f64_e64 s13, v[82:83], 32
-; GFX10-NEXT:    v_cmp_o_f64_e64 s6, v[24:25], v[70:71]
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v48, v16, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v49, v17, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v36, 0x7ff80000, v113, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v35, 0, v35, s5
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s7, v[26:27], v[80:81]
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, v32, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v66, v20, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v33, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v38, v18, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v39, v19, s14
-; GFX10-NEXT:    v_cmp_o_f64_e64 s15, v[26:27], v[80:81]
-; GFX10-NEXT:    v_cndmask_b32_e64 v33, 0, v32, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, v71, v25, s4
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s16, v[28:29], v[84:85]
-; GFX10-NEXT:    v_cmp_o_f64_e64 s8, v[28:29], v[84:85]
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v34, v21, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v33, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 32
-; GFX10-NEXT:    v_cmp_lt_f64_e64 s17, v[30:31], v[86:87]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s5, v[70:71], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v82, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v82, v70, v24, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v19, v83, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v83, 0x7ff80000, v112, s6
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[68:69], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[96:97]
-; GFX10-NEXT:    v_cndmask_b32_e64 v82, 0, v82, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v37, v81, v27, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v80, v26, s7
-; GFX10-NEXT:    v_cmp_class_f64_e64 s6, v[80:81], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s7, v[84:85], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, 0, v[98:99]
-; GFX10-NEXT:    v_cndmask_b32_e64 v113, 0x7ff80000, v37, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v112, 0, v32, s15
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[100:101]
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, v85, v29, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, v84, v28, s16
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s12, 0, v[102:103]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[54:55]
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v35, v22, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v36, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[24:25], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v115, 0x7ff80000, v115, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v114, 0, v114, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, v87, v31, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v32, v86, v30, s17
-; GFX10-NEXT:    v_cmp_class_f64_e64 s8, v[86:87], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s14, 0, v[64:65]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s15, 0, v[50:51]
-; GFX10-NEXT:    v_cndmask_b32_e64 v117, 0x7ff80000, v116, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v116, 0, v32, s18
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s16, 0, v[52:53]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[48:49]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s18, 0, v[38:39]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[33:34]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s20, 0, v[35:36]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s21, 0, v[82:83]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[112:113]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[114:115]
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[116:117]
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v22, v68, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v23, v69, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v82, v24, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v83, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[26:27], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v24, v70, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v25, v71, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v54, v8, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v64, v10, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v50, v12, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v52, v14, s16
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v48, v16, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v38, v18, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v22, v35, v22, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v82, v24, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s11
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v55, v9, s13
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v65, v11, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v51, v13, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v53, v15, s16
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v112, v26, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v113, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[28:29], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v49, v17, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v19, v39, v19, s18
-; GFX10-NEXT:    v_cndmask_b32_e64 v26, v26, v80, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v27, v27, v81, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v23, v36, v23, s20
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v83, v25, s21
-; GFX10-NEXT:    v_cndmask_b32_e64 v26, v112, v26, s22
-; GFX10-NEXT:    v_cndmask_b32_e64 v27, v113, v27, s22
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v114, v28, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v115, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[30:31], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v28, v84, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v29, v85, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v114, v28, s23
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v115, v29, s23
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v116, v30, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v117, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[66:67], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v30, v30, v86, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v31, v31, v87, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v30, v116, v30, s24
-; GFX10-NEXT:    v_cndmask_b32_e64 v31, v117, v31, s24
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v66, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v67, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v20, v33, v20, s19
-; GFX10-NEXT:    v_cndmask_b32_e64 v21, v34, v21, s19
+; GFX10-NEXT:    v_min_f64 v[80:81], v[30:31], v[6:7]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s18, v[30:31], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v83, 0x7ff80000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v84, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v85, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v32, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v33, 0x7ff80000, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v28, v70, 0, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v29, v71, 0x7ff80000, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v26, v66, 0, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v30, v80, 0, s18
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v81, 0x7ff80000, s18
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v16f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_b32 v87, off, s32 offset:8
-; GFX11-NEXT:    scratch_load_b32 v86, off, s32 offset:4
-; GFX11-NEXT:    scratch_load_b32 v85, off, s32 offset:16
-; GFX11-NEXT:    scratch_load_b32 v84, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_b32 v33, off, s32 offset:24
-; GFX11-NEXT:    scratch_load_b32 v32, off, s32 offset:20
-; GFX11-NEXT:    scratch_load_b32 v35, off, s32 offset:32
-; GFX11-NEXT:    scratch_load_b32 v34, off, s32 offset:28
-; GFX11-NEXT:    scratch_load_b32 v37, off, s32 offset:40
-; GFX11-NEXT:    scratch_load_b32 v36, off, s32 offset:36
-; GFX11-NEXT:    scratch_load_b32 v39, off, s32 offset:48
-; GFX11-NEXT:    scratch_load_b32 v38, off, s32 offset:44
-; GFX11-NEXT:    scratch_load_b32 v49, off, s32 offset:56
-; GFX11-NEXT:    scratch_load_b32 v48, off, s32 offset:52
-; GFX11-NEXT:    scratch_load_b32 v51, off, s32 offset:64
-; GFX11-NEXT:    scratch_load_b32 v50, off, s32 offset:60
-; GFX11-NEXT:    scratch_load_b32 v53, off, s32 offset:72
-; GFX11-NEXT:    scratch_load_b32 v52, off, s32 offset:68
-; GFX11-NEXT:    scratch_load_b32 v55, off, s32 offset:80
-; GFX11-NEXT:    scratch_load_b32 v54, off, s32 offset:76
-; GFX11-NEXT:    scratch_load_b32 v65, off, s32 offset:88
-; GFX11-NEXT:    scratch_load_b32 v64, off, s32 offset:84
-; GFX11-NEXT:    scratch_load_b32 v67, off, s32 offset:96
-; GFX11-NEXT:    scratch_load_b32 v66, off, s32 offset:92
-; GFX11-NEXT:    scratch_load_b32 v69, off, s32 offset:104
-; GFX11-NEXT:    scratch_load_b32 v68, off, s32 offset:100
-; GFX11-NEXT:    scratch_load_b32 v71, off, s32 offset:112
-; GFX11-NEXT:    scratch_load_b32 v70, off, s32 offset:108
-; GFX11-NEXT:    scratch_load_b32 v81, off, s32 offset:120
-; GFX11-NEXT:    scratch_load_b32 v80, off, s32 offset:116
 ; GFX11-NEXT:    scratch_load_b32 v31, off, s32
-; GFX11-NEXT:    scratch_load_b32 v83, off, s32 offset:128
-; GFX11-NEXT:    scratch_load_b32 v82, off, s32 offset:124
-; GFX11-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s9, v[0:1], v[86:87]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s11, v[0:1], v[86:87]
-; GFX11-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s10, v[2:3], v[84:85]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s14, v[86:87], 32
-; GFX11-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s0, v[4:5], v[32:33]
-; GFX11-NEXT:    v_cmp_o_f64_e32 vcc_lo, v[4:5], v[32:33]
-; GFX11-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s2, v[6:7], v[34:35]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s12, v[2:3], v[84:85]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s1, v[6:7], v[34:35]
-; GFX11-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s4, v[8:9], v[36:37]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s3, v[8:9], v[36:37]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, v[84:85], 32
-; GFX11-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s6, v[10:11], v[38:39]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s5, v[10:11], v[38:39]
-; GFX11-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s8, v[12:13], v[48:49]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s7, v[12:13], v[48:49]
-; GFX11-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s13, v[14:15], v[50:51]
-; GFX11-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-NEXT:    v_cmp_o_f64_e64 s15, v[16:17], v[52:53]
-; GFX11-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s17, v[18:19], v[54:55]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s18, v[18:19], v[54:55]
-; GFX11-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s19, v[20:21], v[64:65]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s20, v[20:21], v[64:65]
-; GFX11-NEXT:    s_waitcnt vmcnt(9)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s21, v[22:23], v[66:67]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s22, v[22:23], v[66:67]
-; GFX11-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s23, v[24:25], v[68:69]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s24, v[24:25], v[68:69]
-; GFX11-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s25, v[26:27], v[70:71]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s26, v[26:27], v[70:71]
-; GFX11-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s27, v[28:29], v[80:81]
-; GFX11-NEXT:    v_cmp_o_f64_e64 s28, v[28:29], v[80:81]
+; GFX11-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-NEXT:    scratch_load_b32 v49, off, s32 offset:40
+; GFX11-NEXT:    scratch_load_b32 v48, off, s32 offset:36
+; GFX11-NEXT:    scratch_load_b32 v51, off, s32 offset:48
+; GFX11-NEXT:    scratch_load_b32 v50, off, s32 offset:44
+; GFX11-NEXT:    scratch_load_b32 v53, off, s32 offset:56
+; GFX11-NEXT:    scratch_load_b32 v52, off, s32 offset:52
+; GFX11-NEXT:    scratch_load_b32 v55, off, s32 offset:64
+; GFX11-NEXT:    scratch_load_b32 v54, off, s32 offset:60
+; GFX11-NEXT:    scratch_load_b32 v65, off, s32 offset:72
+; GFX11-NEXT:    scratch_load_b32 v64, off, s32 offset:68
+; GFX11-NEXT:    scratch_load_b32 v67, off, s32 offset:80
+; GFX11-NEXT:    scratch_load_b32 v66, off, s32 offset:76
+; GFX11-NEXT:    scratch_load_b32 v69, off, s32 offset:88
+; GFX11-NEXT:    scratch_load_b32 v68, off, s32 offset:84
+; GFX11-NEXT:    scratch_load_b32 v71, off, s32 offset:96
+; GFX11-NEXT:    scratch_load_b32 v70, off, s32 offset:92
+; GFX11-NEXT:    scratch_load_b32 v81, off, s32 offset:104
+; GFX11-NEXT:    scratch_load_b32 v80, off, s32 offset:100
+; GFX11-NEXT:    scratch_load_b32 v83, off, s32 offset:112
+; GFX11-NEXT:    scratch_load_b32 v82, off, s32 offset:108
+; GFX11-NEXT:    scratch_load_b32 v85, off, s32 offset:120
+; GFX11-NEXT:    scratch_load_b32 v84, off, s32 offset:116
+; GFX11-NEXT:    scratch_load_b32 v87, off, s32 offset:128
+; GFX11-NEXT:    scratch_load_b32 v86, off, s32 offset:124
+; GFX11-NEXT:    s_waitcnt vmcnt(30)
+; GFX11-NEXT:    v_min_f64 v[96:97], v[0:1], v[32:33]
+; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33]
+; GFX11-NEXT:    s_waitcnt vmcnt(28)
+; GFX11-NEXT:    v_min_f64 v[32:33], v[2:3], v[34:35]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, v[2:3], v[34:35]
+; GFX11-NEXT:    s_waitcnt vmcnt(26)
+; GFX11-NEXT:    v_min_f64 v[34:35], v[4:5], v[36:37]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s1, v[4:5], v[36:37]
+; GFX11-NEXT:    s_waitcnt vmcnt(24)
+; GFX11-NEXT:    v_min_f64 v[36:37], v[6:7], v[38:39]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s2, v[6:7], v[38:39]
+; GFX11-NEXT:    s_waitcnt vmcnt(22)
+; GFX11-NEXT:    v_min_f64 v[38:39], v[8:9], v[48:49]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s3, v[8:9], v[48:49]
+; GFX11-NEXT:    s_waitcnt vmcnt(20)
+; GFX11-NEXT:    v_min_f64 v[48:49], v[10:11], v[50:51]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s4, v[10:11], v[50:51]
+; GFX11-NEXT:    s_waitcnt vmcnt(18)
+; GFX11-NEXT:    v_min_f64 v[50:51], v[12:13], v[52:53]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s5, v[12:13], v[52:53]
+; GFX11-NEXT:    s_waitcnt vmcnt(16)
+; GFX11-NEXT:    v_min_f64 v[52:53], v[14:15], v[54:55]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s6, v[14:15], v[54:55]
+; GFX11-NEXT:    s_waitcnt vmcnt(14)
+; GFX11-NEXT:    v_min_f64 v[54:55], v[16:17], v[64:65]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s7, v[16:17], v[64:65]
+; GFX11-NEXT:    s_waitcnt vmcnt(12)
+; GFX11-NEXT:    v_min_f64 v[64:65], v[18:19], v[66:67]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s8, v[18:19], v[66:67]
+; GFX11-NEXT:    s_waitcnt vmcnt(10)
+; GFX11-NEXT:    v_min_f64 v[66:67], v[20:21], v[68:69]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s9, v[20:21], v[68:69]
+; GFX11-NEXT:    s_waitcnt vmcnt(8)
+; GFX11-NEXT:    v_min_f64 v[68:69], v[22:23], v[70:71]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s10, v[22:23], v[70:71]
+; GFX11-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-NEXT:    v_min_f64 v[70:71], v[24:25], v[80:81]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s11, v[24:25], v[80:81]
+; GFX11-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-NEXT:    v_min_f64 v[80:81], v[26:27], v[82:83]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s12, v[26:27], v[82:83]
+; GFX11-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NEXT:    v_min_f64 v[82:83], v[28:29], v[84:85]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s13, v[28:29], v[84:85]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s29, v[30:31], v[82:83]
-; GFX11-NEXT:    v_cmp_o_f64_e64 vcc_hi, v[30:31], v[82:83]
-; GFX11-NEXT:    v_cndmask_b32_e64 v96, v87, v1, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v101, v86, v0, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v98, v85, v3, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v103, v84, v2, s10
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[0:1], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v97, 0x7ff80000, v96, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v96, 0, v101, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v100, v33, v5, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v102, v35, v7, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v99, 0x7ff80000, v98, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v98, 0, v103, s12
-; GFX11-NEXT:    v_cmp_class_f64_e64 s11, v[2:3], 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v101, 0x7ff80000, v100, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v103, 0x7ff80000, v102, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v112, v37, v9, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v114, v39, v11, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v116, v49, v13, s8
-; GFX11-NEXT:    v_cmp_o_f64_e64 s9, v[14:15], v[50:51]
-; GFX11-NEXT:    v_cndmask_b32_e64 v118, v51, v15, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v113, 0x7ff80000, v112, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v115, 0x7ff80000, v114, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v117, 0x7ff80000, v116, s7
-; GFX11-NEXT:    v_cmp_lt_f64_e64 s12, v[16:17], v[52:53]
-; GFX11-NEXT:    v_cndmask_b32_e64 v130, v55, v19, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v132, v65, v21, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v134, v67, v23, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, v69, v25, s23
-; GFX11-NEXT:    v_cndmask_b32_e64 v145, v71, v27, s25
-; GFX11-NEXT:    v_cndmask_b32_e64 v131, 0x7ff80000, v130, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v133, 0x7ff80000, v132, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v135, 0x7ff80000, v134, s22
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, v81, v29, s27
-; GFX11-NEXT:    v_cndmask_b32_e64 v148, v80, v28, s27
-; GFX11-NEXT:    v_cndmask_b32_e64 v147, v83, v31, s29
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v147, 0x7ff80000, v147, vcc_hi
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s10
-; GFX11-NEXT:    v_cmp_class_f64_e64 s10, v[36:37], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v86, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v32, v4, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v87, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v34, v6, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s11
-; GFX11-NEXT:    v_cndmask_b32_e32 v100, 0, v86, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[4:5], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v102, 0, v87, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v84, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v36, v8, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v38, v10, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v48, v12, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v119, 0x7ff80000, v118, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v128, v53, v17, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v112, 0, v84, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v114, 0, v86, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v116, 0, v87, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v50, v14, s13
-; GFX11-NEXT:    v_cndmask_b32_e64 v129, 0x7ff80000, v128, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v52, v16, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v54, v18, s17
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v85, s16
-; GFX11-NEXT:    v_cndmask_b32_e64 v118, 0, v84, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, v64, v20, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v128, 0, v86, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v130, 0, v87, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, v66, v22, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v85, 0x7ff80000, v144, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v132, 0, v84, s20
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, v68, v24, s23
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, v70, v26, s25
-; GFX11-NEXT:    v_cndmask_b32_e64 v134, 0, v86, s22
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[68:69], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s1, v[70:71], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v84, 0, v87, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v87, 0x7ff80000, v145, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v86, 0, v144, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v145, 0x7ff80000, v146, s28
-; GFX11-NEXT:    v_cndmask_b32_e64 v144, 0, v148, s28
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, v82, v30, s29
-; GFX11-NEXT:    v_cmp_class_f64_e64 s2, v[80:81], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s3, v[82:83], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s6, v[32:33], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, v[34:35], 32
-; GFX11-NEXT:    v_dual_cndmask_b32 v5, v101, v5 :: v_dual_cndmask_b32 v4, v100, v4
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[6:7], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v146, 0, v146, vcc_hi
-; GFX11-NEXT:    v_cmp_class_f64_e64 s12, v[38:39], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s14, v[48:49], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s16, v[50:51], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s18, v[52:53], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s20, v[54:55], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s21, v[64:65], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s4, 0, v[96:97]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[98:99]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s7, 0, v[100:101]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s9, 0, v[102:103]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s11, 0, v[112:113]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s13, 0, v[114:115]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s15, 0, v[116:117]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s17, 0, v[118:119]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s19, 0, v[128:129]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s22, 0, v[130:131]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s23, 0, v[132:133]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s24, 0, v[134:135]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s25, 0, v[84:85]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s26, 0, v[86:87]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s27, 0, v[144:145]
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s28, 0, v[146:147]
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v33, s6
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v103, v7 :: v_dual_cndmask_b32 v6, v102, v6
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[8:9], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v32, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v35, s8
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, v0, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v98, v2, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v100, v4, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, v1, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v99, v3, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v101, v5, s7
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v103, v7, s9
-; GFX11-NEXT:    v_dual_cndmask_b32 v9, v113, v9 :: v_dual_cndmask_b32 v8, v112, v8
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[10:11], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v34, s8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v37, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v102, v6, s9
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v113, v9, s11
-; GFX11-NEXT:    v_dual_cndmask_b32 v11, v115, v11 :: v_dual_cndmask_b32 v10, v114, v10
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[12:13], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v36, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v39, s12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v112, v8, s11
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v115, v11, s13
-; GFX11-NEXT:    v_dual_cndmask_b32 v13, v117, v13 :: v_dual_cndmask_b32 v12, v116, v12
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[14:15], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v38, s12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v49, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v114, v10, s13
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v117, v13, s15
-; GFX11-NEXT:    v_dual_cndmask_b32 v15, v119, v15 :: v_dual_cndmask_b32 v14, v118, v14
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[16:17], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v48, s14
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v51, s16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v116, v12, s15
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v119, v15, s17
-; GFX11-NEXT:    v_dual_cndmask_b32 v17, v129, v17 :: v_dual_cndmask_b32 v16, v128, v16
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[18:19], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v50, s16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v17, v53, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v118, v14, s17
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v17, v129, v17, s19
-; GFX11-NEXT:    v_dual_cndmask_b32 v19, v131, v19 :: v_dual_cndmask_b32 v18, v130, v18
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[20:21], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v16, v52, s18
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v19, v55, s20
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v16, v128, v16, s19
-; GFX11-NEXT:    v_cndmask_b32_e64 v19, v131, v19, s22
-; GFX11-NEXT:    v_dual_cndmask_b32 v21, v133, v21 :: v_dual_cndmask_b32 v20, v132, v20
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[22:23], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v18, v54, s20
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v21, v65, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v18, v130, v18, s22
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v21, v133, v21, s23
-; GFX11-NEXT:    v_dual_cndmask_b32 v23, v135, v23 :: v_dual_cndmask_b32 v22, v134, v22
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[24:25], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v20, v64, s21
-; GFX11-NEXT:    v_cndmask_b32_e64 v20, v132, v20, s23
-; GFX11-NEXT:    v_dual_cndmask_b32 v25, v85, v25 :: v_dual_cndmask_b32 v24, v84, v24
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[26:27], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v25, v25, v69, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v25, v85, v25, s25
-; GFX11-NEXT:    v_dual_cndmask_b32 v27, v87, v27 :: v_dual_cndmask_b32 v26, v86, v26
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[28:29], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v24, v68, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v27, v27, v71, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v24, v84, v24, s25
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v27, v87, v27, s26
-; GFX11-NEXT:    v_dual_cndmask_b32 v29, v145, v29 :: v_dual_cndmask_b32 v28, v144, v28
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[30:31], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v26, v26, v70, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v29, v29, v81, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v26, v86, v26, s26
-; GFX11-NEXT:    v_cndmask_b32_e64 v29, v145, v29, s27
-; GFX11-NEXT:    v_dual_cndmask_b32 v31, v147, v31 :: v_dual_cndmask_b32 v30, v146, v30
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[66:67], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v28, v28, v80, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v31, v31, v83, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v28, v144, v28, s27
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v31, v147, v31, s28
-; GFX11-NEXT:    v_dual_cndmask_b32 v23, v23, v67 :: v_dual_cndmask_b32 v22, v22, v66
-; GFX11-NEXT:    v_cndmask_b32_e64 v30, v30, v82, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v23, v135, v23, s24
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v22, v134, v22, s24
-; GFX11-NEXT:    v_cndmask_b32_e64 v30, v146, v30, s28
+; GFX11-NEXT:    v_min_f64 v[84:85], v[30:31], v[86:87]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s14, v[30:31], v[86:87]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v96, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v97, 0x7ff80000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v32, 0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v33, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v34, 0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v35, 0x7ff80000, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v36, 0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v37, 0x7ff80000, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v8, v38, 0, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v39, 0x7ff80000, s3
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v48, 0, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v49, 0x7ff80000, s4
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v50, 0, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v51, 0x7ff80000, s5
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v52, 0, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v53, 0x7ff80000, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v16, v54, 0, s7
+; GFX11-NEXT:    v_cndmask_b32_e64 v17, v55, 0x7ff80000, s7
+; GFX11-NEXT:    v_cndmask_b32_e64 v18, v64, 0, s8
+; GFX11-NEXT:    v_cndmask_b32_e64 v19, v65, 0x7ff80000, s8
+; GFX11-NEXT:    v_cndmask_b32_e64 v20, v66, 0, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v21, v67, 0x7ff80000, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v22, v68, 0, s10
+; GFX11-NEXT:    v_cndmask_b32_e64 v23, v69, 0x7ff80000, s10
+; GFX11-NEXT:    v_cndmask_b32_e64 v24, v70, 0, s11
+; GFX11-NEXT:    v_cndmask_b32_e64 v25, v71, 0x7ff80000, s11
+; GFX11-NEXT:    v_cndmask_b32_e64 v26, v80, 0, s12
+; GFX11-NEXT:    v_cndmask_b32_e64 v27, v81, 0x7ff80000, s12
+; GFX11-NEXT:    v_cndmask_b32_e64 v28, v82, 0, s13
+; GFX11-NEXT:    v_cndmask_b32_e64 v29, v83, 0x7ff80000, s13
+; GFX11-NEXT:    v_cndmask_b32_e64 v30, v84, 0, s14
+; GFX11-NEXT:    v_cndmask_b32_e64 v31, v85, 0x7ff80000, s14
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v16f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll b/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll
index ab035b9de04b9d3d1c91ec48abaac4b5e1204832..826862e1249203f5eef204c26da804f200c65a65 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll
@@ -1,63 +1,160 @@
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefixes=SI,GCN,SI-NOHSA,FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefixes=VI,VI-NOHSA,GCN,FUNC %s
-; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=EG,FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn < %s | FileCheck --check-prefixes=SI,GCN,FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga  < %s | FileCheck --check-prefixes=VI,GCN,FUNC %s
+; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=R600,FUNC %s
 
-
-; FUNC-LABEL: {{^}}local_size_x:
-; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]]
-; EG: MOV * [[VAL]], KC0[1].Z
-
-; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x6
-; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x18
-; CI-HSA: s_load_dword [[XY:s[0-9]+]], s[4:5], 0x1
-; VI-HSA: s_load_dword [[XY:s[0-9]+]], s[4:5], 0x4
-
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_x(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_x:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x6
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_x:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x18
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_x:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MOV * T1.X, KC0[1].Z,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %0 = call i32 @llvm.r600.read.local.size.x() #0
   store i32 %0, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_y:
-; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]]
-; EG: MOV * [[VAL]], KC0[1].W
-
-; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x7
-; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x1c
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_y(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_y:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x7
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_y:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x1c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_y:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MOV * T1.X, KC0[1].W,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %0 = call i32 @llvm.r600.read.local.size.y() #0
   store i32 %0, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_z:
-; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]]
-; EG: MOV * [[VAL]], KC0[2].X
-
-; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x8
-; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x20
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_z(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_z:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x8
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_z:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x20
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_z:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MOV * T1.X, KC0[2].X,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %0 = call i32 @llvm.r600.read.local.size.z() #0
   store i32 %0, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_xy:
-; SI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9+]]]], s[0:1], 0x6
-; VI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9+]]]], s[0:1], 0x18
-; GCN: s_mul_i32 [[VAL:s[0-9]+]], s[[X]], s[[Y]]
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_xy(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_xy:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x6
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mul_i32 s4, s4, s5
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_xy:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x18
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mul_i32 s2, s2, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_xy:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MULLO_INT * T1.X, KC0[1].Z, KC0[1].W,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %x = call i32 @llvm.r600.read.local.size.x() #0
   %y = call i32 @llvm.r600.read.local.size.y() #0
@@ -66,17 +163,43 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_xz:
-
-; SI-NOHSA-DAG: s_load_dword [[X:s[0-9]+]], s[0:1], 0x6
-; SI-NOHSA-DAG: s_load_dword [[Z:s[0-9]+]], s[0:1], 0x8
-; VI-NOHSA-DAG: s_load_dword [[X:s[0-9]+]], s[0:1], 0x18
-; VI-NOHSA-DAG: s_load_dword [[Z:s[0-9]+]], s[0:1], 0x20
-; HSA-DAG: s_and_b32 [[X:s[0-9]+]], [[XY]], 0xffff
-; GCN: s_mul_i32 [[VAL:s[0-9]+]], [[X]], [[Z]]
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_xz(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_xz:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s2, s[0:1], 0x6
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x8
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mul_i32 s4, s2, s4
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_xz:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x18
+; VI-NEXT:    s_load_dword s3, s[0:1], 0x20
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mul_i32 s2, s2, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_xz:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MULLO_INT * T1.X, KC0[1].Z, KC0[2].X,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %x = call i32 @llvm.r600.read.local.size.x() #0
   %z = call i32 @llvm.r600.read.local.size.z() #0
@@ -85,16 +208,41 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_yz:
-; HSA: enable_sgpr_private_segment_buffer = 1
-; HSA: enable_sgpr_dispatch_ptr = 1
-
-; SI-NOHSA-DAG: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s[0:1], 0x7
-; VI-NOHSA-DAG: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s[0:1], 0x1c
-; GCN: s_mul_i32 [[VAL:s[0-9]+]], s[[#LOAD + 0]], s[[#LOAD + 1]]
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_yz(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_yz:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x7
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mul_i32 s0, s0, s1
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s2
+; SI-NEXT:    s_mov_b32 s5, s3
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_yz:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x1c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mul_i32 s0, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_yz:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     MULLO_INT * T1.X, KC0[1].W, KC0[2].X,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %y = call i32 @llvm.r600.read.local.size.y() #0
   %z = call i32 @llvm.r600.read.local.size.z() #0
@@ -103,19 +251,46 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_xyz:
-; HSA: enable_sgpr_private_segment_buffer = 1
-; HSA: enable_sgpr_dispatch_ptr = 1
-
-; SI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9]+]]], s[0:1], 0x6
-; SI-NOHSA-DAG: s_load_dword s[[Z:[0-9]+]], s[0:1], 0x8
-; VI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9]+]]], s[0:1], 0x18
-; VI-NOHSA-DAG: s_load_dword s[[Z:[0-9]+]], s[0:1], 0x20
-; GCN: s_mul_i32 [[M:s[0-9]+]], s[[X]], s[[Y]]
-; GCN: s_add_i32 [[VAL:s[0-9]+]], [[M]], s[[Z]]
-; GCN-DAG: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_xyz(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_xyz:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x6
+; SI-NEXT:    s_load_dword s2, s[0:1], 0x8
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mul_i32 s4, s4, s5
+; SI-NEXT:    s_add_i32 s4, s4, s2
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_xyz:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x18
+; VI-NEXT:    s_load_dword s4, s[0:1], 0x20
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mul_i32 s2, s2, s3
+; VI-NEXT:    s_add_i32 s2, s2, s4
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_xyz:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MULLO_INT * T0.X, KC0[1].Z, KC0[1].W,
+; R600-NEXT:     ADD_INT T0.X, PS, KC0[2].X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %x = call i32 @llvm.r600.read.local.size.x() #0
   %y = call i32 @llvm.r600.read.local.size.y() #0
@@ -126,13 +301,39 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_x_known_bits:
-; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x6
-; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x18
-; GCN-NOT: 0xffff
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN-NEXT: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_x_known_bits(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_x_known_bits:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x6
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_x_known_bits:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x18
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_x_known_bits:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     AND_INT * T1.X, KC0[1].Z, literal.y,
+; R600-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
 entry:
   %size = call i32 @llvm.r600.read.local.size.x() #0
   %shl = shl i32 %size, 16
@@ -141,13 +342,39 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_y_known_bits:
-; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x7
-; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x1c
-; GCN-NOT: 0xffff
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN-NEXT: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_y_known_bits(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_y_known_bits:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x7
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_y_known_bits:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x1c
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_y_known_bits:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     AND_INT * T1.X, KC0[1].W, literal.y,
+; R600-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
 entry:
   %size = call i32 @llvm.r600.read.local.size.y() #0
   %shl = shl i32 %size, 16
@@ -156,13 +383,39 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}local_size_z_known_bits:
-; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x8
-; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x20
-; GCN-NOT: 0xffff
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]]
-; GCN-NEXT: buffer_store_dword [[VVAL]]
 define amdgpu_kernel void @local_size_z_known_bits(ptr addrspace(1) %out) {
+; SI-LABEL: local_size_z_known_bits:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x8
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: local_size_z_known_bits:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x20
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; R600-LABEL: local_size_z_known_bits:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     AND_INT * T1.X, KC0[2].X, literal.y,
+; R600-NEXT:    2(2.802597e-45), 65535(9.183409e-41)
 entry:
   %size = call i32 @llvm.r600.read.local.size.z() #0
   %shl = shl i32 %size, 16
@@ -176,3 +429,6 @@ declare i32 @llvm.r600.read.local.size.y() #0
 declare i32 @llvm.r600.read.local.size.z() #0
 
 attributes #0 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FUNC: {{.*}}
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
index d87eb9711488c307e9d896990677dada692d363e..e0ccda1cbb9eddd914abe15a507b4024db5854f2 100644
--- a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
+++ b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
@@ -13,6 +13,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W64,GFX1030,GFX1030W64 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1151 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W32 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1151 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W64 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1152 < %s | FileCheck --check-prefixes=GCN,GFX1030,GFX1030W32 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1152 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1030,GFX1030W64 %s
 
 ; GCN-LABEL: {{^}}max_occupancy:
 ; GFX9:       ; Occupancy: 10
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index 4d6adc7cc94171f63f6f691966ec9ae70e302026..a3f7906a05f6b199548369b934bbed6ff0690dbd 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -1,17 +1,21 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}fma_vector_vector_scalar_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @fma_vector_vector_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_scalar_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -28,18 +32,21 @@ bb:
 }
 
 ; Apply fneg to broadcasted vector
-; GCN-LABEL: {{^}}fma_vector_vector_neg_broadcast_scalar_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_broadcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_neg_broadcast_scalar_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -57,18 +64,21 @@ bb:
 }
 
 ; Apply fneg before broadcast
-; GCN-LABEL: {{^}}fma_vector_vector_neg_scalar_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_neg_scalar_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -86,18 +96,21 @@ bb:
 }
 
 ; Apply fneg before and after broadcast, and should cancel out.
-; GCN-LABEL: {{^}}fma_vector_vector_neg_broadcast_neg_scalar_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_broadcast_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_neg_broadcast_neg_scalar_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -116,18 +129,21 @@ bb:
 }
 
 ; Add scalar, but negate low component
-; GCN-LABEL: {{^}}fma_vector_vector_scalar_neg_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_scalar_neg_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_scalar_neg_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -144,18 +160,21 @@ bb:
 }
 
 ; Add scalar, but negate high component
-; GCN-LABEL: {{^}}fma_vector_vector_scalar_neg_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_scalar_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_scalar_neg_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
 
@@ -172,17 +191,20 @@ bb:
 }
 
 ; Apply fneg before broadcast with bitcast
-; GCN-LABEL: {{^}}add_vector_neg_bitcast_scalar_lo:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_add_u16 v{{[0-9]+}}, [[VEC0]], [[SCALAR0]] op_sel_hi:[1,0] neg_lo:[0,1] neg_hi:[0,1]{{$}}
 define amdgpu_kernel void @add_vector_neg_bitcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: add_vector_neg_bitcast_scalar_lo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v0, v0
+; GCN-NEXT:    ds_read_u16 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_add_u16 v0, v0, v1 op_sel_hi:[1,0] neg_lo:[0,1] neg_hi:[0,1]
+; GCN-NEXT:    global_store_dword v2, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
@@ -197,19 +219,26 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_scalar_lo_neg_scalar_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR1:v[0-9]+]]
-
-; FIXME: Remove and
-; GCN-DAG: v_and_b32_e32 [[SCALAR0]], 0xffff, [[SCALAR0]]
-; GCN-DAG: v_xor_b32_e32 [[SCALAR1]], 0x8000, [[SCALAR1]]
-; GCN: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[SCALAR1]], 16, [[SCALAR0]]
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]]{{$}}
 define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_scalar_lo_neg_scalar_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v3, v1
+; GCN-NEXT:    ds_read_u16 v1, v1 offset:4
+; GCN-NEXT:    s_waitcnt lgkmcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GCN-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v1
+; GCN-NEXT:    global_store_dword v4, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %arg2.gep = getelementptr inbounds half, ptr addrspace(3) %arg2, i32 2
@@ -229,15 +258,23 @@ bb:
 }
 
 ; FIXME: Can we avoid waitcnt between the two halves?
-; GCN-LABEL: {{^}}fma_vector_vector_neg_scalar_lo_scalar_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[PACKED:v[0-9]+]]
-; GCN: s_waitcnt
-; GCN: ds_read_u16_d16_hi [[PACKED]]
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
+; GCN-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    ds_read_u16 v3, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_u16_d16_hi v3, v1 offset:4
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v2, v0, v3 neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v1, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %arg2.gep = getelementptr inbounds half, ptr addrspace(3) %arg2, i32 2
@@ -257,18 +294,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_neg_vector_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_vector_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_neg_vector_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -285,18 +325,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_vector_neg_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_vector_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_vector_neg_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -314,17 +357,20 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}add_vector_scalar_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_add_u16 v{{[0-9]+}}, [[VEC0]], [[VEC1]] op_sel:[0,1]{{$}}
 define amdgpu_kernel void @add_vector_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: add_vector_scalar_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_add_u16 v0, v1, v0 op_sel:[0,1]
+; GCN-NEXT:    global_store_dword v2, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x i16>, ptr addrspace(3) %lds, i32 1
 
@@ -338,18 +384,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_scalar_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_scalar_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -366,18 +415,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_neg_vector_lo_neg_hi:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_vector_lo_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_neg_vector_lo_neg_hi:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -396,18 +448,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_swap_vector:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @fma_vector_vector_swap_vector(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_swap_vector:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -423,19 +478,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_swap_neg_vector:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-; GCN-NOT: xor
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_swap_neg_vector(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_swap_neg_vector:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -452,19 +509,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_0:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-; GCN-NOT: xor
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_0(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_0:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -480,19 +539,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_1:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-; GCN-NOT: xor
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_lo:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_1(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_1:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 neg_lo:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -508,19 +569,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_2:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-; GCN-NOT: xor
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_2(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_2:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 neg_hi:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -536,19 +599,21 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_3:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: or
-; GCN-NOT: xor
-
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] neg_lo:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_3(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_3:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] neg_lo:[0,0,1]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -564,9 +629,22 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}bitcast_fneg_f32:
-; GCN: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+$}}
 define amdgpu_kernel void @bitcast_fneg_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: bitcast_fneg_f32:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT:    v_pk_add_f16 v0, v0, v1
+; GCN-NEXT:    global_store_dword v2, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %vec0 = load volatile <2 x half>, ptr addrspace(3) %lds, align 4
   %f32 = load volatile float, ptr addrspace(3) undef, align 4
@@ -578,9 +656,22 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}shuffle_bitcast_fneg_f32:
-; GCN: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} op_sel:[0,1] op_sel_hi:[1,0]{{$}}
 define amdgpu_kernel void @shuffle_bitcast_fneg_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: shuffle_bitcast_fneg_f32:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT:    v_pk_add_f16 v0, v0, v1 op_sel:[0,1] op_sel_hi:[1,0]
+; GCN-NEXT:    global_store_dword v2, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %vec0 = load volatile <2 x half>, ptr addrspace(3) %lds, align 4
 
@@ -593,10 +684,24 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}extract_from_i64:
-; GCN: v_lshl_or_b32
-; GCN: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+$}}
 define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: extract_from_i64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v2, v0
+; GCN-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GCN-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_pk_add_u16 v0, v2, v0
+; GCN-NEXT:    global_store_dword v1, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4
   %i64 = load volatile i64, ptr addrspace(1) undef
@@ -612,21 +717,24 @@ bb:
   ret void
 }
 
-
-; Bitcast is final obstacle to identifying same source register
-; GCN-LABEL: {{^}}bitcast_lo_elt_op_sel:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: _or
-
-; GCN: v_pk_add_f16 [[FADD:v[0-9]+]]
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @bitcast_lo_elt_op_sel(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: bitcast_lo_elt_op_sel:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_ushort v3, v[0:1], off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0]
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
@@ -647,21 +755,29 @@ bb:
   ret void
 }
 
-
-; Bitcast is final obstacle to identifying same source register
-; GCN-LABEL: {{^}}mix_elt_types_op_sel:
-; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_b32 [[VEC2:v[0-9]+]]
-
-; GCN-NOT: pack
-; GCN-NOT: and
-; GCN-NOT: shl
-; GCN-NOT: _or
-
-; GCN: v_pk_add_f16 [[FADD:v[0-9]+]]
-; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}}
 define amdgpu_kernel void @mix_elt_types_op_sel(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
+; GCN-LABEL: mix_elt_types_op_sel:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    ds_read_b32 v1, v0
+; GCN-NEXT:    ds_read_b32 v2, v0 offset:4
+; GCN-NEXT:    ds_read_b32 v0, v0 offset:8
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    ; kill: killed $vgpr0_vgpr1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_ushort v3, v[0:1], off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_ushort v3, v[0:1], off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; kill: killed $vgpr0_vgpr1
+; GCN-NEXT:    v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
+; GCN-NEXT:    global_store_dword v3, v0, s[0:1]
+; GCN-NEXT:    s_endpgm
 bb:
   %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
   %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll b/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
index 462ab38b4cd58dbd75c516a6b43863f32e9c844e..1f7de7343efdb7d205cfa237f6ca160104312e55 100644
--- a/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
+++ b/llvm/test/CodeGen/AMDGPU/private-memory-r600.ll
@@ -12,11 +12,11 @@ declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 ; R600: LDS_READ
 ; R600: LDS_READ
 
-; OPT: call i32 @llvm.r600.read.local.size.y(), !range !0
-; OPT: call i32 @llvm.r600.read.local.size.z(), !range !0
-; OPT: call i32 @llvm.r600.read.tidig.x(), !range !1
-; OPT: call i32 @llvm.r600.read.tidig.y(), !range !1
-; OPT: call i32 @llvm.r600.read.tidig.z(), !range !1
+; OPT: call range(i32 0, 257) i32 @llvm.r600.read.local.size.y()
+; OPT: call range(i32 0, 257) i32 @llvm.r600.read.local.size.z()
+; OPT: call range(i32 0, 256) i32 @llvm.r600.read.tidig.x()
+; OPT: call range(i32 0, 256) i32 @llvm.r600.read.tidig.y()
+; OPT: call range(i32 0, 256) i32 @llvm.r600.read.tidig.z()
 
 define amdgpu_kernel void @mova_same_clause(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %in) #0 {
 entry:
@@ -276,7 +276,4 @@ define amdgpu_kernel void @ptrtoint(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
   ret void
 }
 
-; OPT: !0 = !{i32 0, i32 257}
-; OPT: !1 = !{i32 0, i32 256}
-
 attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-strip-abi-opt-attributes.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-strip-abi-opt-attributes.ll
index ada1b841cd67c3ffa7aa3ef81de403285423ab4e..778fe904382f2f192fd0391ddc9f045f1e04fa94 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-strip-abi-opt-attributes.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-strip-abi-opt-attributes.ll
@@ -5,9 +5,9 @@
 
 ; CHECK-LABEL: define amdgpu_kernel void @promote_to_lds(ptr addrspace(1) %out, i32 %in) #0 {
 ; CHECK: call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK: call i32 @llvm.amdgcn.workitem.id.x(), !range !2
-; CHECK: call i32 @llvm.amdgcn.workitem.id.y(), !range !2
-; CHECK: call i32 @llvm.amdgcn.workitem.id.z(), !range !2
+; CHECK: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK: call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
 define amdgpu_kernel void @promote_to_lds(ptr addrspace(1) %out, i32 %in) #0 {
 entry:
   %tmp = alloca [2 x i32], addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
index efc11bf1a606da0f3d1adfe5bd360ef7915bf38e..f56656270c335d3f757876e7fb284726f2dc470d 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll
@@ -8,22 +8,22 @@
 define amdgpu_kernel void @lds_promoted_alloca_icmp_same_derived_pointer(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
 ; CHECK-LABEL: @lds_promoted_alloca_icmp_same_derived_pointer(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(4) [[TMP3]], align 4, !invariant.load !0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(4) [[TMP5]], align 4, !range [[RNG1:![0-9]+]], !invariant.load !0
-; CHECK-NEXT:    [[TMP7:%.*]] = lshr i32 [[TMP4]], 16
-; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.workitem.id.x(), !range [[RNG2:![0-9]+]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.amdgcn.workitem.id.y(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.workitem.id.z(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw nsw i32 [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw nsw i32 [[TMP9]], [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_same_derived_pointer.alloca, i32 0, i32 [[TMP15]]
-; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP16]], i32 0, i32 [[A:%.*]]
-; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP16]], i32 0, i32 [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !range [[RNG1:![0-9]+]], !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP3]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[TMP8:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[TMP9:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw nsw i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw nsw i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_same_derived_pointer.alloca, i32 0, i32 [[TMP14]]
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP15]], i32 0, i32 [[A:%.*]]
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP15]], i32 0, i32 [[B:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(3) [[PTR0]], [[PTR1]]
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    store volatile i32 [[ZEXT]], ptr addrspace(1) [[OUT:%.*]], align 4
@@ -50,21 +50,21 @@ define amdgpu_kernel void @lds_promoted_alloca_icmp_same_derived_pointer(ptr add
 define amdgpu_kernel void @lds_promoted_alloca_icmp_null_rhs(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
 ; CHECK-LABEL: @lds_promoted_alloca_icmp_null_rhs(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(4) [[TMP3]], align 4, !invariant.load !0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(4) [[TMP5]], align 4, !range [[RNG1]], !invariant.load !0
-; CHECK-NEXT:    [[TMP7:%.*]] = lshr i32 [[TMP4]], 16
-; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.workitem.id.x(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.amdgcn.workitem.id.y(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.workitem.id.z(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw nsw i32 [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw nsw i32 [[TMP9]], [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_null_rhs.alloca, i32 0, i32 [[TMP15]]
-; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP16]], i32 0, i32 [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !range [[RNG1]], !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP3]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[TMP8:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[TMP9:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw nsw i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw nsw i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_null_rhs.alloca, i32 0, i32 [[TMP14]]
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP15]], i32 0, i32 [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(3) [[PTR0]], null
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    store volatile i32 [[ZEXT]], ptr addrspace(1) [[OUT:%.*]], align 4
@@ -89,21 +89,21 @@ define amdgpu_kernel void @lds_promoted_alloca_icmp_null_rhs(ptr addrspace(1) %o
 define amdgpu_kernel void @lds_promoted_alloca_icmp_null_lhs(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
 ; CHECK-LABEL: @lds_promoted_alloca_icmp_null_lhs(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call noalias nonnull dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(4) [[TMP3]], align 4, !invariant.load !0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(4) [[TMP5]], align 4, !range [[RNG1]], !invariant.load !0
-; CHECK-NEXT:    [[TMP7:%.*]] = lshr i32 [[TMP4]], 16
-; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.workitem.id.x(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.amdgcn.workitem.id.y(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.workitem.id.z(), !range [[RNG2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw nsw i32 [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw nsw i32 [[TMP9]], [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_null_lhs.alloca, i32 0, i32 [[TMP15]]
-; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP16]], i32 0, i32 [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(4) [[TMP1]], i64 2
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !range [[RNG1]], !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP3]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[TMP8:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[TMP9:%.*]] = call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw nsw i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw nsw i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [256 x [16 x i32]], ptr addrspace(3) @lds_promoted_alloca_icmp_null_lhs.alloca, i32 0, i32 [[TMP14]]
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(3) [[TMP15]], i32 0, i32 [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(3) null, [[PTR0]]
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    store volatile i32 [[ZEXT]], ptr addrspace(1) [[OUT:%.*]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir
index 4ecce2842455bdc0ff1012258a160270c5a6fae5..6dda1fe1f39da547bc0394f28d573a2d67131ec9 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir
@@ -282,3 +282,168 @@ body:             |
     %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
     FLAT_STORE_DWORD %13, %0.sub1, 0, 0, implicit $exec, implicit $flat_scr
 ...
+
+---
+# GCN-LABEL: name: diffoporder_add_global_atomic_cmpswap
+# GFX9: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 1000, 0,
+# GFX9: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+
+# GFX8: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+# GFX8: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+
+name: diffoporder_add_global_atomic_cmpswap
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    GLOBAL_ATOMIC_CMPSWAP %6:vreg_64, %0:vreg_64, 0, 0, implicit $exec
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    GLOBAL_ATOMIC_CMPSWAP %13:vreg_64, %0:vreg_64, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: diffoporder_add_flat_atomic_cmpswap
+# GFX9: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 1000, 0,
+# GFX9: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+
+# GFX8: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+# GFX8: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0,
+
+name: diffoporder_add_flat_atomic_cmpswap
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    FLAT_ATOMIC_CMPSWAP %6:vreg_64, %0:vreg_64, 0, 0, implicit $exec, implicit $flat_scr
+
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    FLAT_ATOMIC_CMPSWAP %13:vreg_64, %0:vreg_64, 0, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+# GCN-LABEL: name: diffoporder_add_global_atomic_add
+# GFX9: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 1000, 0,
+# GFX9: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+
+# GFX8: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+# GFX8: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+
+name: diffoporder_add_global_atomic_add
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    GLOBAL_ATOMIC_ADD %6:vreg_64, %0.sub0, 0, 0, implicit $exec
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    GLOBAL_ATOMIC_ADD %13:vreg_64, %0.sub0, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: diffoporder_add_flat_atomic_add
+# GFX9: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 1000, 0,
+# GFX9: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+
+# GFX8: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+# GFX8: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0,
+
+name: diffoporder_add_flat_atomic_add
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    FLAT_ATOMIC_ADD %6:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr
+
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    FLAT_ATOMIC_ADD %13:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+# GCN-LABEL: name: diffoporder_add_global_atomic_add_rtn
+# GFX9: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 1000, 0,
+# GFX9: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+
+# GFX8: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+# GFX8: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+
+name: diffoporder_add_global_atomic_add_rtn
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    %14:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN %6:vreg_64, %0.sub0, 0, 0, implicit $exec
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    %15:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN %13:vreg_64, %0.sub0, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: diffoporder_add_flat_atomic_add_rtn
+# GFX9: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 1000, 0,
+# GFX9: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+
+# GFX8: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+# GFX8: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0,
+
+name: diffoporder_add_flat_atomic_add_rtn
+body:             |
+  bb.0.entry:
+
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+
+    %1:sgpr_32 = S_MOV_B32 4000
+    %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec
+    %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec
+    %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1
+    %14:vgpr_32 = FLAT_ATOMIC_ADD_RTN %6:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr
+
+
+    %8:sgpr_32 = S_MOV_B32 3000
+    %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec
+    %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec
+    %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+    %15:vgpr_32 = FLAT_ATOMIC_ADD_RTN %13:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index b068d87c4d6f48e058ccfc724f8079836d1bae27..6b036f675929e5fb8b40db529562f2c4c0ff4185 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -10,29 +10,31 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-LABEL: v_srem_i128_vv:
 ; GFX9:       ; %bb.0: ; %_udiv-special-cases
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, 0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v20, 31, v3
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v20
-; GFX9-NEXT:    v_xor_b32_e32 v10, v2, v20
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v20
-; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v0, v20
-; GFX9-NEXT:    v_xor_b32_e32 v9, v3, v20
-; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v1, v20, vcc
-; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GFX9-NEXT:    v_subb_co_u32_e32 v0, vcc, v10, v20, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v4, v4, v8
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v9, v20, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v5, v5, v8
-; GFX9-NEXT:    v_sub_co_u32_e32 v23, vcc, v4, v8
-; GFX9-NEXT:    v_xor_b32_e32 v6, v6, v8
-; GFX9-NEXT:    v_subb_co_u32_e32 v21, vcc, v5, v8, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v6, v8, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v8, vcc
-; GFX9-NEXT:    v_or_b32_e32 v7, v21, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, 0, v4
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v21, v20
+; GFX9-NEXT:    v_cndmask_b32_e32 v22, v5, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v23, v4, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v7, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v10, vcc
+; GFX9-NEXT:    v_or_b32_e32 v7, v22, v5
 ; GFX9-NEXT:    v_or_b32_e32 v6, v23, v4
 ; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
-; GFX9-NEXT:    v_or_b32_e32 v7, v3, v1
-; GFX9-NEXT:    v_or_b32_e32 v6, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v7, v1, v3
+; GFX9-NEXT:    v_or_b32_e32 v6, v0, v2
 ; GFX9-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[6:7]
 ; GFX9-NEXT:    v_ffbh_u32_e32 v6, v4
 ; GFX9-NEXT:    v_add_u32_e32 v6, 32, v6
@@ -40,38 +42,37 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_min_u32_e32 v6, v6, v7
 ; GFX9-NEXT:    v_ffbh_u32_e32 v7, v23
 ; GFX9-NEXT:    v_add_u32_e32 v7, 32, v7
-; GFX9-NEXT:    v_ffbh_u32_e32 v8, v21
+; GFX9-NEXT:    v_ffbh_u32_e32 v8, v22
 ; GFX9-NEXT:    v_min_u32_e32 v7, v7, v8
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, 64, v7
 ; GFX9-NEXT:    v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX9-NEXT:    v_ffbh_u32_e32 v9, v1
+; GFX9-NEXT:    v_ffbh_u32_e32 v10, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT:    v_ffbh_u32_e32 v7, v0
+; GFX9-NEXT:    v_ffbh_u32_e32 v7, v2
 ; GFX9-NEXT:    v_add_u32_e32 v7, 32, v7
-; GFX9-NEXT:    v_min_u32_e32 v7, v7, v9
-; GFX9-NEXT:    v_ffbh_u32_e32 v9, v2
-; GFX9-NEXT:    v_add_u32_e32 v9, 32, v9
-; GFX9-NEXT:    v_ffbh_u32_e32 v10, v3
-; GFX9-NEXT:    v_min_u32_e32 v9, v9, v10
+; GFX9-NEXT:    v_min_u32_e32 v7, v7, v10
+; GFX9-NEXT:    v_ffbh_u32_e32 v10, v0
+; GFX9-NEXT:    v_add_u32_e32 v10, 32, v10
+; GFX9-NEXT:    v_ffbh_u32_e32 v11, v1
+; GFX9-NEXT:    v_min_u32_e32 v10, v10, v11
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, v8, 0, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, 64, v9
-; GFX9-NEXT:    v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
-; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0x7f
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, 0, vcc
-; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v8, v10, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 64, v10
+; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[6:7], 0, 0, vcc
+; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, 0, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, v6, v7
+; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v8, v11, vcc
 ; GFX9-NEXT:    v_subbrev_co_u32_e32 v8, vcc, 0, v9, vcc
 ; GFX9-NEXT:    v_subbrev_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0x7f
 ; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
 ; GFX9-NEXT:    v_or_b32_e32 v13, v7, v9
 ; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v22, v20
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc
@@ -82,10 +83,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
 ; GFX9-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v1, 0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v0, 0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v3, 0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v2, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v3, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, v2, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v1, 0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v0, 0, s[4:5]
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_6
@@ -98,22 +99,22 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_sub_u32_e32 v11, 64, v13
 ; GFX9-NEXT:    v_or_b32_e32 v8, v25, v27
 ; GFX9-NEXT:    v_or_b32_e32 v7, v24, v26
-; GFX9-NEXT:    v_lshlrev_b64 v[9:10], v13, v[0:1]
-; GFX9-NEXT:    v_lshrrev_b64 v[11:12], v11, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[9:10], v13, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[11:12], v11, v[0:1]
 ; GFX9-NEXT:    v_sub_u32_e32 v6, 63, v6
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
-; GFX9-NEXT:    v_lshlrev_b64 v[6:7], v6, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], v6, v[0:1]
 ; GFX9-NEXT:    v_or_b32_e32 v8, v10, v12
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v11
 ; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v13
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v13
-; GFX9-NEXT:    v_lshlrev_b64 v[12:13], v13, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[12:13], v13, v[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v1, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v0, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v3, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v2, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, v13, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v11, 0
@@ -123,23 +124,23 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_5
 ; GFX9-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GFX9-NEXT:    v_sub_u32_e32 v10, 64, v24
-; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v24, v[2:3]
-; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v10, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v24, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v10, v[2:3]
 ; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v24
 ; GFX9-NEXT:    v_or_b32_e32 v10, v8, v10
 ; GFX9-NEXT:    v_subrev_u32_e32 v8, 64, v24
 ; GFX9-NEXT:    v_or_b32_e32 v11, v9, v11
-; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v8, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v8, v[2:3]
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v24
 ; GFX9-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v15, v9, v3, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v15, v9, v1, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v8, v10, vcc
-; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v24, v[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v10, v2, s[4:5]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v24, v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v10, v0, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v17, 0, v9, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v16, 0, v8, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, -1, v23
-; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, -1, v21, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, -1, v22, vcc
 ; GFX9-NEXT:    v_addc_co_u32_e32 v30, vcc, -1, v4, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v18, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v10, 0
@@ -168,7 +169,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_or_b32_e32 v12, v18, v12
 ; GFX9-NEXT:    v_and_b32_e32 v18, v8, v23
 ; GFX9-NEXT:    v_or_b32_e32 v13, v19, v13
-; GFX9-NEXT:    v_and_b32_e32 v19, v8, v21
+; GFX9-NEXT:    v_and_b32_e32 v19, v8, v22
 ; GFX9-NEXT:    v_sub_co_u32_e32 v14, vcc, v14, v18
 ; GFX9-NEXT:    v_and_b32_e32 v32, v8, v4
 ; GFX9-NEXT:    v_subb_co_u32_e32 v15, vcc, v15, v19, vcc
@@ -207,7 +208,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_mov_b32_e32 v15, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v14, v6
-; GFX9-NEXT:    v_mad_u64_u32 v[13:14], s[4:5], v21, v13, v[14:15]
+; GFX9-NEXT:    v_mad_u64_u32 v[13:14], s[4:5], v22, v13, v[14:15]
 ; GFX9-NEXT:    v_mul_lo_u32 v9, v10, v4
 ; GFX9-NEXT:    v_mul_lo_u32 v11, v11, v23
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v14
@@ -218,272 +219,283 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-NEXT:    v_mov_b32_e32 v8, v14
 ; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v4, v8
 ; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v12, v12, v21
-; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v21, v10, v[8:9]
+; GFX9-NEXT:    v_mul_lo_u32 v12, v12, v22
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v22, v10, v[8:9]
 ; GFX9-NEXT:    v_add3_u32 v4, v11, v7, v12
 ; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v8, v6
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v4, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v7, v13
-; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v2, v5
-; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v7, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v0, vcc, v0, v6, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v5, v0, v20
-; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v20
-; GFX9-NEXT:    v_xor_b32_e32 v4, v1, v22
-; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v22
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v5
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v4, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v20
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v21
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v20
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v22, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v5, v20, vcc
-; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v4, v22, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v20
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v21, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v21
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v20, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v21, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-O0-LABEL: v_srem_i128_vv:
 ; GFX9-O0:       ; %bb.0: ; %_udiv-special-cases
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-O0-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $vgpr8 : SGPR spill to VGPR lane
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v2
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v4
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v0
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v2
+; GFX9-O0-NEXT:    ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v19, v7
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v7
+; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v9
+; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v3
+; GFX9-O0-NEXT:    ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v3
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 63
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
-; GFX9-O0-NEXT:    v_ashrrev_i64 v[11:12], s4, v[10:11]
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v13
+; GFX9-O0-NEXT:    v_ashrrev_i64 v[3:4], s4, v[3:4]
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v11
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v6
-; GFX9-O0-NEXT:    v_ashrrev_i64 v[15:16], s4, v[13:14]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v12
-; GFX9-O0-NEXT:    v_xor_b32_e64 v3, v3, v10
-; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 killed $vgpr8_vgpr9 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v11
-; GFX9-O0-NEXT:    v_xor_b32_e64 v13, v8, v12
-; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v2
-; GFX9-O0-NEXT:    v_xor_b32_e64 v3, v3, v10
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_xor_b32_e64 v1, v1, v12
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v16
-; GFX9-O0-NEXT:    v_xor_b32_e64 v9, v8, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v15
-; GFX9-O0-NEXT:    v_xor_b32_e64 v7, v7, v6
-; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v5
-; GFX9-O0-NEXT:    v_xor_b32_e64 v9, v9, v3
-; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, v6
-; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v14
-; GFX9-O0-NEXT:    v_sub_co_u32_e32 v9, vcc, v9, v12
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v10, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v13, vcc, v11, v12, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v10, vcc
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v13
+; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 0
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 1
+; GFX9-O0-NEXT:    s_mov_b32 s10, s6
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s10, 2
+; GFX9-O0-NEXT:    s_mov_b32 s11, s7
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s11, 3
+; GFX9-O0-NEXT:    v_sub_co_u32_e32 v10, vcc, s10, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v6, vcc, v5, v4, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s10
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v1, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v7, vcc, v7, v2, vcc
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v2
+; GFX9-O0-NEXT:    ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v11
+; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_lt_i64_e64 s[4:5], v[12:13], s[4:5]
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v10
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[4:5]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v4
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[4:5]
+; GFX9-O0-NEXT:    ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v4
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v8
-; GFX9-O0-NEXT:    v_sub_co_u32_e32 v1, vcc, v1, v6
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v4, vcc, v4, v3, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v11, vcc, v5, v6, vcc
-; GFX9-O0-NEXT:    v_subb_co_u32_e32 v3, vcc, v2, v3, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v9
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v18
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v19
+; GFX9-O0-NEXT:    v_sub_co_u32_e32 v14, vcc, s10, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v12, vcc, v11, v10, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s10
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v11, vcc, v11, v8, vcc
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, s11
+; GFX9-O0-NEXT:    v_subb_co_u32_e32 v13, vcc, v13, v9, vcc
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v4
+; GFX9-O0-NEXT:    ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v15, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v15
+; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_lt_i64_e64 s[4:5], v[18:19], s[4:5]
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v14
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v12, s[4:5]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v19, v10
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v13
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v12
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v9, v13, s[4:5]
+; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v8, v8, v11, s[4:5]
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
-; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v3
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v14
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v9
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v5
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v17
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v11
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v18
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v19
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v11
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v18
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v19
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v14
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v5
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v17
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v2
-; GFX9-O0-NEXT:    v_or_b32_e64 v3, v8, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v11
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
-; GFX9-O0-NEXT:    v_or_b32_e64 v1, v5, v6
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 0
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 0
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 1
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v14
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT:    v_or_b32_e64 v15, v4, v2
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v13
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v9
-; GFX9-O0-NEXT:    v_or_b32_e64 v9, v3, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v15
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v19
+; GFX9-O0-NEXT:    v_or_b32_e64 v15, v13, v14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v11
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v18
+; GFX9-O0-NEXT:    v_or_b32_e64 v13, v13, v14
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v15
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[4:5], v[13:14], s[6:7]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v6
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT:    v_or_b32_e64 v15, v13, v14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT:    v_or_b32_e64 v13, v13, v14
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v15
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[13:14], s[6:7]
 ; GFX9-O0-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v6, v6
-; GFX9-O0-NEXT:    s_mov_b32 s9, 32
-; GFX9-O0-NEXT:    v_add_u32_e64 v6, v6, s9
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v7, v7
-; GFX9-O0-NEXT:    v_min_u32_e64 v6, v6, v7
-; GFX9-O0-NEXT:    s_mov_b32 s8, 0
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v7
-; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v5
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[11:12], s[8:9]
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v8, v8
-; GFX9-O0-NEXT:    v_min_u32_e64 v15, v5, v8
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v5
-; GFX9-O0-NEXT:    s_mov_b64 s[10:11], 64
-; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v15
-; GFX9-O0-NEXT:    s_mov_b32 s12, s10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v16
-; GFX9-O0-NEXT:    s_mov_b32 s14, s11
-; GFX9-O0-NEXT:    v_add_co_u32_e64 v8, s[12:13], v8, s12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s14
-; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13]
+; GFX9-O0-NEXT:    s_mov_b32 s13, 32
+; GFX9-O0-NEXT:    v_add_u32_e64 v8, v8, s13
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v9, v9
+; GFX9-O0-NEXT:    v_min_u32_e64 v8, v8, v9
+; GFX9-O0-NEXT:    s_mov_b32 s12, 0
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
-; GFX9-O0-NEXT:    s_mov_b64 s[12:13], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13]
-; GFX9-O0-NEXT:    v_cndmask_b32_e64 v5, v5, v10, s[12:13]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v8
-; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v6, v7, s[12:13]
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v11
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v9
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v7, v7
+; GFX9-O0-NEXT:    v_add_u32_e64 v7, v7, s13
+; GFX9-O0-NEXT:    v_ffbh_u32_e64 v10, v10
+; GFX9-O0-NEXT:    v_min_u32_e64 v13, v7, v10
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, s12
+; GFX9-O0-NEXT:    ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v14, v7
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], 64
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v13
+; GFX9-O0-NEXT:    s_mov_b32 s16, s14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v14
+; GFX9-O0-NEXT:    s_mov_b32 s18, s15
+; GFX9-O0-NEXT:    v_add_co_u32_e64 v10, s[16:17], v10, s16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, s18
+; GFX9-O0-NEXT:    v_addc_co_u32_e64 v7, s[16:17], v7, v11, s[16:17]
+; GFX9-O0-NEXT:    ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v11
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v7, v7, v12, s[8:9]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v10
+; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, v8, v9, s[8:9]
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v7
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[5:6], s[8:9]
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v1
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s13
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v6, v2
 ; GFX9-O0-NEXT:    v_min_u32_e64 v6, v5, v6
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr12
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr16
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v5
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v7
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v5, v3
-; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s9
+; GFX9-O0-NEXT:    v_add_u32_e64 v5, v5, s13
 ; GFX9-O0-NEXT:    v_ffbh_u32_e64 v11, v4
-; GFX9-O0-NEXT:    v_min_u32_e64 v15, v5, v11
-; GFX9-O0-NEXT:    ; implicit-def: $sgpr9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s8
-; GFX9-O0-NEXT:    ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v16, v5
-; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v15
-; GFX9-O0-NEXT:    s_mov_b32 s8, s10
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v16
-; GFX9-O0-NEXT:    s_mov_b32 s10, s11
-; GFX9-O0-NEXT:    v_add_co_u32_e64 v11, s[8:9], v11, s8
-; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s10
-; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9]
+; GFX9-O0-NEXT:    v_min_u32_e64 v12, v5, v11
+; GFX9-O0-NEXT:    ; implicit-def: $sgpr13
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, s12
+; GFX9-O0-NEXT:    ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v11, v12
+; GFX9-O0-NEXT:    s_mov_b32 s12, s14
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v13
+; GFX9-O0-NEXT:    s_mov_b32 s14, s15
+; GFX9-O0-NEXT:    v_add_co_u32_e64 v11, s[12:13], v11, s12
+; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s14
+; GFX9-O0-NEXT:    v_addc_co_u32_e64 v5, s[12:13], v5, v12, s[12:13]
 ; GFX9-O0-NEXT:    ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v12, v5
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v12
-; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[8:9]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v6
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v11
@@ -496,8 +508,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v10
-; GFX9-O0-NEXT:    s_mov_b32 s10, s6
-; GFX9-O0-NEXT:    s_mov_b32 s11, s7
 ; GFX9-O0-NEXT:    v_sub_co_u32_e32 v5, vcc, v5, v8
 ; GFX9-O0-NEXT:    v_subb_co_u32_e32 v9, vcc, v6, v7, vcc
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, s10
@@ -580,75 +590,75 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[4:5], exec
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s4, 2
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s5, 3
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s4, 4
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s5, 5
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execz .LBB0_3
 ; GFX9-O0-NEXT:    s_branch .LBB0_8
 ; GFX9-O0-NEXT:  .LBB0_1: ; %Flow
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v0, 4
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v0, 5
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v0, 6
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v0, 7
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:  ; %bb.2: ; %Flow
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_5
 ; GFX9-O0-NEXT:  .LBB0_3: ; %Flow2
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v4, 2
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v4, 3
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v4, 4
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v4, 5
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_9
 ; GFX9-O0-NEXT:  .LBB0_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 1
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-O0-NEXT:    v_lshlrev_b64 v[2:3], s4, v[0:1]
@@ -681,67 +691,67 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_3
 ; GFX9-O0-NEXT:  .LBB0_5: ; %Flow1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s4, v8, 6
-; GFX9-O0-NEXT:    v_readlane_b32 s5, v8, 7
+; GFX9-O0-NEXT:    v_readlane_b32 s4, v8, 8
+; GFX9-O0-NEXT:    v_readlane_b32 s5, v8, 9
 ; GFX9-O0-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_4
 ; GFX9-O0-NEXT:  .LBB0_6: ; %udiv-do-while
 ; GFX9-O0-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    v_readlane_b32 s6, v16, 8
-; GFX9-O0-NEXT:    v_readlane_b32 s7, v16, 9
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT:    v_readlane_b32 s6, v16, 10
+; GFX9-O0-NEXT:    v_readlane_b32 s7, v16, 11
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 63
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(16)
 ; GFX9-O0-NEXT:    v_lshrrev_b64 v[29:30], s4, v[2:3]
@@ -881,72 +891,72 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v3
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v2
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v1
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v0
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v15
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v14
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v18, v13
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 4
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 5
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 6
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 7
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], s[4:5]
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 8
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 9
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s6, 10
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s7, 11
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execnz .LBB0_6
 ; GFX9-O0-NEXT:    s_branch .LBB0_1
 ; GFX9-O0-NEXT:  .LBB0_7: ; %udiv-preheader
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
@@ -1027,51 +1037,51 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v13, v17
 ; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[4:5], s[6:7]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v15, s9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v14, s8
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v13, s7
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s4, 8
-; GFX9-O0-NEXT:    v_writelane_b32 v16, s5, 9
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s4, 10
+; GFX9-O0-NEXT:    v_writelane_b32 v16, s5, 11
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_branch .LBB0_6
 ; GFX9-O0-NEXT:  .LBB0_8: ; %udiv-bb1
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -1108,14 +1118,14 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v1
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v10
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 0x7f
 ; GFX9-O0-NEXT:    v_sub_u32_e64 v3, s4, v4
 ; GFX9-O0-NEXT:    v_lshlrev_b64 v[5:6], v3, v[11:12]
@@ -1161,12 +1171,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr4
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v3
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v10
 ; GFX9-O0-NEXT:    v_or_b32_e64 v3, v3, v4
@@ -1181,33 +1191,33 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v4, s9
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, s7
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-O0-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; GFX9-O0-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 6
-; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 7
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s6, 8
+; GFX9-O0-NEXT:    v_writelane_b32 v0, s7, 9
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_cbranch_execz .LBB0_5
 ; GFX9-O0-NEXT:    s_branch .LBB0_7
 ; GFX9-O0-NEXT:  .LBB0_9: ; %udiv-end
-; GFX9-O0-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT:    s_or_saveexec_b64 s[22:23], -1
 ; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT:    s_mov_b64 exec, s[22:23]
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
@@ -1218,10 +1228,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b32 s4, 32
@@ -1495,11 +1505,11 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX9-O0-NEXT:    ; kill: killed $vgpr4
 ; GFX9-O0-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index ea30a63b0be19ac2f682d1801cd89e8f66616235..6372d74161fad74c567584fac5083ce8d6bd31d8 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -28,34 +28,33 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
-; GCN-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
+; GCN-NEXT:    v_max_i32_e32 v2, v1, v2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v5
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v5
+; GCN-NEXT:    v_max_i32_e32 v5, v0, v5
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_xor_b32_e32 v2, v5, v2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, v3
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v0, v1
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GCN-NEXT:    v_mul_hi_u32 v3, v5, v3
+; GCN-NEXT:    v_mul_lo_u32 v1, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v5, v1
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v1, v2
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v2
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v0
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
@@ -73,34 +72,33 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; TONGA-NEXT:    s_mov_b32 s0, s4
 ; TONGA-NEXT:    s_mov_b32 s1, s5
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v2
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v1
-; TONGA-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
+; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, 0, v1
+; TONGA-NEXT:    v_max_i32_e32 v2, v1, v2
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v3, v2
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v2
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, 0, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v5
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v5
+; TONGA-NEXT:    v_max_i32_e32 v5, v0, v5
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
+; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; TONGA-NEXT:    v_xor_b32_e32 v2, v5, v2
 ; TONGA-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; TONGA-NEXT:    v_mul_hi_u32 v4, v3, v4
 ; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
-; TONGA-NEXT:    v_mul_hi_u32 v3, v0, v3
-; TONGA-NEXT:    v_mul_lo_u32 v4, v3, v1
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v3
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
-; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, v0, v1
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; TONGA-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; TONGA-NEXT:    v_mul_hi_u32 v3, v5, v3
+; TONGA-NEXT:    v_mul_lo_u32 v1, v3, v2
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
+; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v5, v1
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, v1, v2
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; TONGA-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v2
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v0
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v1, v0
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
@@ -115,40 +113,37 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GFX9-NEXT:    s_mov_b32 s8, s6
 ; GFX9-NEXT:    s_mov_b32 s9, s7
 ; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX9-NEXT:    s_ashr_i32 s6, s0, 31
-; GFX9-NEXT:    s_add_i32 s0, s0, s6
-; GFX9-NEXT:    s_xor_b32 s7, s0, s6
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v1
+; GFX9-NEXT:    s_abs_i32 s7, s6
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s7
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX9-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX9-NEXT:    s_xor_b32 s5, s4, s6
+; GFX9-NEXT:    s_sub_i32 s6, 0, s7
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
-; GFX9-NEXT:    s_xor_b32 s6, s5, s6
-; GFX9-NEXT:    s_xor_b32 s4, s4, s5
+; GFX9-NEXT:    s_abs_i32 s4, s4
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s5, 0, s7
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
-; GFX9-NEXT:    s_mul_i32 s5, s5, s8
-; GFX9-NEXT:    s_mul_hi_u32 s5, s8, s5
-; GFX9-NEXT:    s_add_i32 s8, s8, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, s8
-; GFX9-NEXT:    s_mul_i32 s8, s5, s7
+; GFX9-NEXT:    s_mul_i32 s6, s6, s8
+; GFX9-NEXT:    s_mul_hi_u32 s6, s8, s6
+; GFX9-NEXT:    s_add_i32 s8, s8, s6
+; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s8
+; GFX9-NEXT:    s_mul_i32 s8, s6, s7
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_add_i32 s9, s5, 1
+; GFX9-NEXT:    s_add_i32 s9, s6, 1
 ; GFX9-NEXT:    s_sub_i32 s8, s4, s7
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX9-NEXT:    s_cselect_b32 s6, s9, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_add_i32 s8, s5, 1
+; GFX9-NEXT:    s_add_i32 s8, s6, 1
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s5
-; GFX9-NEXT:    s_xor_b32 s4, s4, s6
-; GFX9-NEXT:    s_sub_i32 s4, s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s6
+; GFX9-NEXT:    s_xor_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
@@ -408,62 +403,60 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; GCN-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
-; GCN-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v5
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v7
-; GCN-NEXT:    v_xor_b32_e32 v8, v4, v5
-; GCN-NEXT:    v_xor_b32_e32 v9, v6, v7
-; GCN-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; GCN-NEXT:    v_sub_i32_e32 v10, vcc, 0, v2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 0, v3
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v2
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
+; GCN-NEXT:    v_xor_b32_e32 v4, v0, v2
+; GCN-NEXT:    v_xor_b32_e32 v7, v1, v3
+; GCN-NEXT:    v_max_i32_e32 v2, v2, v6
+; GCN-NEXT:    v_max_i32_e32 v3, v3, v9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v6, v2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v9, v3
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GCN-NEXT:    v_max_i32_e32 v0, v0, v5
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v9
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 0, v2
+; GCN-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GCN-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
-; GCN-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GCN-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GCN-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_sub_i32_e32 v10, vcc, 0, v3
+; GCN-NEXT:    v_mul_lo_u32 v9, v9, v6
 ; GCN-NEXT:    v_mul_lo_u32 v10, v10, v5
-; GCN-NEXT:    v_mul_lo_u32 v11, v11, v7
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v6
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v4, v5, v10
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v7, v11
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v1
+; GCN-NEXT:    v_mul_hi_u32 v9, v6, v9
+; GCN-NEXT:    v_max_i32_e32 v1, v1, v8
+; GCN-NEXT:    v_mul_hi_u32 v8, v5, v10
+; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v4
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v4, v2
+; GCN-NEXT:    v_ashrrev_i32_e32 v7, 31, v7
+; GCN-NEXT:    v_mul_lo_u32 v8, v6, v2
 ; GCN-NEXT:    v_mul_lo_u32 v10, v5, v3
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v6
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
 ; GCN-NEXT:    v_add_i32_e32 v11, vcc, 1, v5
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v2
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v1, v3
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e32 v8, vcc, v0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, v1, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[2:3]
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v6, v8, vcc
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v8
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v9
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v4
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v7
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
@@ -481,62 +474,60 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; TONGA-NEXT:    s_mov_b32 s4, s0
 ; TONGA-NEXT:    s_mov_b32 s5, s1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; TONGA-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
-; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v3, v7
-; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
-; TONGA-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v5
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v7
-; TONGA-NEXT:    v_xor_b32_e32 v8, v4, v5
-; TONGA-NEXT:    v_xor_b32_e32 v9, v6, v7
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; TONGA-NEXT:    v_sub_u32_e32 v10, vcc, 0, v2
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; TONGA-NEXT:    v_sub_u32_e32 v11, vcc, 0, v3
+; TONGA-NEXT:    v_sub_u32_e32 v6, vcc, 0, v2
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, 0, v3
+; TONGA-NEXT:    v_xor_b32_e32 v4, v0, v2
+; TONGA-NEXT:    v_xor_b32_e32 v7, v1, v3
+; TONGA-NEXT:    v_max_i32_e32 v2, v2, v6
+; TONGA-NEXT:    v_max_i32_e32 v3, v3, v9
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v6, v2
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v9, v3
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, 0, v0
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; TONGA-NEXT:    v_max_i32_e32 v0, v0, v5
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v5, v9
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, 0, v2
+; TONGA-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; TONGA-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
-; TONGA-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v10, vcc, 0, v3
+; TONGA-NEXT:    v_mul_lo_u32 v9, v9, v6
 ; TONGA-NEXT:    v_mul_lo_u32 v10, v10, v5
-; TONGA-NEXT:    v_mul_lo_u32 v11, v11, v7
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v1, v6
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v4
-; TONGA-NEXT:    v_mul_hi_u32 v4, v5, v10
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v6
-; TONGA-NEXT:    v_mul_hi_u32 v6, v7, v11
-; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v7, v6
-; TONGA-NEXT:    v_mul_hi_u32 v4, v0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v8, vcc, 0, v1
+; TONGA-NEXT:    v_mul_hi_u32 v9, v6, v9
+; TONGA-NEXT:    v_max_i32_e32 v1, v1, v8
+; TONGA-NEXT:    v_mul_hi_u32 v8, v5, v10
+; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v4
+; TONGA-NEXT:    v_add_u32_e32 v6, vcc, v6, v9
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v5, v8
+; TONGA-NEXT:    v_mul_hi_u32 v6, v0, v6
 ; TONGA-NEXT:    v_mul_hi_u32 v5, v1, v5
-; TONGA-NEXT:    v_mul_lo_u32 v6, v4, v2
+; TONGA-NEXT:    v_ashrrev_i32_e32 v7, 31, v7
+; TONGA-NEXT:    v_mul_lo_u32 v8, v6, v2
 ; TONGA-NEXT:    v_mul_lo_u32 v10, v5, v3
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, 1, v4
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v6
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, 1, v6
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v8
 ; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v10
 ; TONGA-NEXT:    v_add_u32_e32 v11, vcc, 1, v5
 ; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v2
 ; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v1, v3
-; TONGA-NEXT:    v_sub_u32_e32 v6, vcc, v0, v2
-; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[0:1]
-; TONGA-NEXT:    v_sub_u32_e32 v7, vcc, v1, v3
+; TONGA-NEXT:    v_sub_u32_e32 v8, vcc, v0, v2
+; TONGA-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[0:1]
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, v1, v3
 ; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[2:3]
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, 1, v4
-; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[2:3]
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, 1, v5
+; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v8, vcc, 1, v6
+; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[2:3]
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, 1, v5
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; TONGA-NEXT:    v_cndmask_b32_e32 v0, v6, v8, vcc
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; TONGA-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v8
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v9
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v8
-; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v9
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v4
+; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v7
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v7
 ; TONGA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; TONGA-NEXT:    s_endpgm
 ;
@@ -553,69 +544,63 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
-; GFX9-NEXT:    s_ashr_i32 s1, s0, 31
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_xor_b32 s6, s0, s1
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT:    s_abs_i32 s1, s0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s1
 ; GFX9-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX9-NEXT:    s_ashr_i32 s8, s7, 31
-; GFX9-NEXT:    s_add_i32 s7, s7, s8
+; GFX9-NEXT:    s_xor_b32 s0, s7, s0
+; GFX9-NEXT:    s_ashr_i32 s8, s0, 31
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    s_xor_b32 s9, s8, s1
-; GFX9-NEXT:    s_xor_b32 s1, s7, s8
-; GFX9-NEXT:    s_sub_i32 s7, 0, s6
+; GFX9-NEXT:    s_sub_i32 s0, 0, s1
+; GFX9-NEXT:    s_abs_i32 s7, s7
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
+; GFX9-NEXT:    s_mul_i32 s0, s0, s9
+; GFX9-NEXT:    s_mul_hi_u32 s0, s9, s0
+; GFX9-NEXT:    s_add_i32 s9, s9, s0
+; GFX9-NEXT:    s_mul_hi_u32 s0, s7, s9
+; GFX9-NEXT:    s_mul_i32 s9, s0, s1
+; GFX9-NEXT:    s_sub_i32 s7, s7, s9
+; GFX9-NEXT:    s_add_i32 s10, s0, 1
+; GFX9-NEXT:    s_sub_i32 s9, s7, s1
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s1
+; GFX9-NEXT:    s_cselect_b32 s0, s10, s0
+; GFX9-NEXT:    s_cselect_b32 s7, s9, s7
+; GFX9-NEXT:    s_add_i32 s9, s0, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s1
+; GFX9-NEXT:    s_cselect_b32 s7, s9, s0
+; GFX9-NEXT:    s_abs_i32 s9, s6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s9
 ; GFX9-NEXT:    s_mov_b32 s0, s4
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX9-NEXT:    s_mov_b32 s1, s5
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_xor_b32 s5, s4, s6
+; GFX9-NEXT:    s_xor_b32 s6, s7, s8
+; GFX9-NEXT:    s_sub_i32 s7, 0, s9
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s6, s6, s8
+; GFX9-NEXT:    s_abs_i32 s4, s4
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
 ; GFX9-NEXT:    s_mul_i32 s7, s7, s8
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s8, s7
 ; GFX9-NEXT:    s_add_i32 s8, s8, s7
-; GFX9-NEXT:    s_mul_hi_u32 s7, s1, s8
-; GFX9-NEXT:    s_mul_i32 s8, s7, s6
-; GFX9-NEXT:    s_sub_i32 s1, s1, s8
+; GFX9-NEXT:    s_mul_hi_u32 s7, s4, s8
+; GFX9-NEXT:    s_mul_i32 s8, s7, s9
+; GFX9-NEXT:    s_sub_i32 s4, s4, s8
 ; GFX9-NEXT:    s_add_i32 s10, s7, 1
-; GFX9-NEXT:    s_sub_i32 s8, s1, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
+; GFX9-NEXT:    s_sub_i32 s8, s4, s9
+; GFX9-NEXT:    s_cmp_ge_u32 s4, s9
 ; GFX9-NEXT:    s_cselect_b32 s7, s10, s7
-; GFX9-NEXT:    s_cselect_b32 s1, s8, s1
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
 ; GFX9-NEXT:    s_add_i32 s8, s7, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s8, s7
-; GFX9-NEXT:    s_ashr_i32 s7, s4, 31
-; GFX9-NEXT:    s_add_i32 s4, s4, s7
-; GFX9-NEXT:    s_xor_b32 s4, s4, s7
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX9-NEXT:    s_ashr_i32 s8, s5, 31
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s6, s6, s9
-; GFX9-NEXT:    s_add_i32 s5, s5, s8
-; GFX9-NEXT:    s_xor_b32 s7, s8, s7
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s6, s6, s9
-; GFX9-NEXT:    s_xor_b32 s5, s5, s8
-; GFX9-NEXT:    s_sub_i32 s8, 0, s4
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
-; GFX9-NEXT:    s_mul_i32 s8, s8, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s9, s8
-; GFX9-NEXT:    s_add_i32 s9, s9, s8
-; GFX9-NEXT:    s_mul_hi_u32 s8, s5, s9
-; GFX9-NEXT:    s_mul_i32 s9, s8, s4
-; GFX9-NEXT:    s_sub_i32 s5, s5, s9
-; GFX9-NEXT:    s_add_i32 s10, s8, 1
-; GFX9-NEXT:    s_sub_i32 s9, s5, s4
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX9-NEXT:    s_add_i32 s9, s8, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s8
-; GFX9-NEXT:    s_xor_b32 s4, s4, s7
-; GFX9-NEXT:    s_sub_i32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s4, s9
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s7
+; GFX9-NEXT:    s_xor_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -819,119 +804,115 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    s_mov_b32 s8, s0
 ; GCN-NEXT:    s_mov_b32 s9, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; GCN-NEXT:    v_sub_i32_e32 v12, vcc, 0, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
-; GCN-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
-; GCN-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; GCN-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v10
-; GCN-NEXT:    v_xor_b32_e32 v4, v4, v9
-; GCN-NEXT:    v_xor_b32_e32 v5, v5, v11
-; GCN-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
-; GCN-NEXT:    v_xor_b32_e32 v15, v8, v9
-; GCN-NEXT:    v_xor_b32_e32 v16, v10, v11
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v8
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v10
-; GCN-NEXT:    v_cvt_f32_u32_e32 v8, v4
-; GCN-NEXT:    v_cvt_f32_u32_e32 v10, v5
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v12
-; GCN-NEXT:    v_xor_b32_e32 v6, v6, v13
-; GCN-NEXT:    v_xor_b32_e32 v17, v12, v13
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v12
-; GCN-NEXT:    v_cvt_f32_u32_e32 v12, v6
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v10, v10
-; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 0, v4
+; GCN-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v5
+; GCN-NEXT:    v_sub_i32_e32 v16, vcc, 0, v6
+; GCN-NEXT:    v_xor_b32_e32 v8, v0, v4
+; GCN-NEXT:    v_xor_b32_e32 v11, v1, v5
+; GCN-NEXT:    v_xor_b32_e32 v14, v2, v6
+; GCN-NEXT:    v_max_i32_e32 v4, v4, v10
+; GCN-NEXT:    v_max_i32_e32 v5, v5, v13
+; GCN-NEXT:    v_max_i32_e32 v6, v6, v16
+; GCN-NEXT:    v_max_i32_e32 v1, v1, v12
+; GCN-NEXT:    v_ashrrev_i32_e32 v10, 31, v14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v12, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v14, v5
+; GCN-NEXT:    v_cvt_f32_u32_e32 v16, v6
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 0, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v12, v12
-; GCN-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
-; GCN-NEXT:    v_mul_f32_e32 v10, 0x4f7ffffe, v10
-; GCN-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; GCN-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v14, v14
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v16, v16
+; GCN-NEXT:    v_sub_i32_e32 v15, vcc, 0, v2
 ; GCN-NEXT:    v_mul_f32_e32 v12, 0x4f7ffffe, v12
+; GCN-NEXT:    v_mul_f32_e32 v14, 0x4f7ffffe, v14
+; GCN-NEXT:    v_mul_f32_e32 v16, 0x4f7ffffe, v16
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v12, v12
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 0, v5
-; GCN-NEXT:    v_mul_lo_u32 v9, v9, v8
-; GCN-NEXT:    v_mul_lo_u32 v11, v11, v10
-; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v6
+; GCN-NEXT:    v_cvt_u32_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_u32_f32_e32 v16, v16
+; GCN-NEXT:    v_sub_i32_e32 v17, vcc, 0, v7
+; GCN-NEXT:    v_max_i32_e32 v0, v0, v9
+; GCN-NEXT:    v_ashrrev_i32_e32 v9, 31, v11
+; GCN-NEXT:    v_max_i32_e32 v2, v2, v15
+; GCN-NEXT:    v_max_i32_e32 v11, v7, v17
+; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v4
+; GCN-NEXT:    v_sub_i32_e32 v15, vcc, 0, v5
+; GCN-NEXT:    v_sub_i32_e32 v17, vcc, 0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v13, v13, v12
-; GCN-NEXT:    v_mul_hi_u32 v9, v8, v9
-; GCN-NEXT:    v_mul_hi_u32 v11, v10, v11
-; GCN-NEXT:    v_ashrrev_i32_e32 v14, 31, v7
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v7, v14
+; GCN-NEXT:    v_mul_lo_u32 v15, v15, v14
+; GCN-NEXT:    v_mul_lo_u32 v17, v17, v16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v18, v11
 ; GCN-NEXT:    v_mul_hi_u32 v13, v12, v13
-; GCN-NEXT:    v_xor_b32_e32 v7, v7, v14
-; GCN-NEXT:    v_cvt_f32_u32_e32 v18, v7
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; GCN-NEXT:    v_add_i32_e32 v9, vcc, v10, v11
-; GCN-NEXT:    v_mul_hi_u32 v8, v0, v8
-; GCN-NEXT:    v_mul_hi_u32 v9, v1, v9
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, v12, v13
-; GCN-NEXT:    v_mul_hi_u32 v10, v2, v10
+; GCN-NEXT:    v_mul_hi_u32 v15, v14, v15
+; GCN-NEXT:    v_mul_hi_u32 v17, v16, v17
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v18, v18
-; GCN-NEXT:    v_mul_lo_u32 v11, v8, v4
-; GCN-NEXT:    v_mul_lo_u32 v13, v9, v5
-; GCN-NEXT:    v_mul_lo_u32 v21, v10, v6
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, v14, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, v16, v17
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v12
+; GCN-NEXT:    v_mul_hi_u32 v13, v1, v13
+; GCN-NEXT:    v_mul_hi_u32 v14, v2, v14
 ; GCN-NEXT:    v_mul_f32_e32 v18, 0x4f7ffffe, v18
+; GCN-NEXT:    v_mul_lo_u32 v15, v12, v4
+; GCN-NEXT:    v_mul_lo_u32 v17, v13, v5
+; GCN-NEXT:    v_mul_lo_u32 v21, v14, v6
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v18, v18
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v11
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v13
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, 1, v8
-; GCN-NEXT:    v_add_i32_e32 v20, vcc, 1, v9
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v15
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v17
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v21
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 1, v12
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 1, v13
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 1, v14
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v4
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v1, v5
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v21
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, v0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v12, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v12, vcc, v1, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v9, v9, v20, s[2:3]
-; GCN-NEXT:    v_sub_i32_e32 v19, vcc, 0, v7
-; GCN-NEXT:    v_add_i32_e32 v22, vcc, 1, v10
-; GCN-NEXT:    v_sub_i32_e32 v13, vcc, v2, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v11, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v11, vcc, 1, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v12, s[2:3]
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, 1, v9
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GCN-NEXT:    v_sub_i32_e32 v19, vcc, 0, v11
+; GCN-NEXT:    v_sub_i32_e32 v17, vcc, v0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v16, vcc, v1, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v13, v13, v20, s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[4:5]
+; GCN-NEXT:    v_mul_lo_u32 v19, v19, v18
+; GCN-NEXT:    v_sub_i32_e32 v20, vcc, v2, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v17, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 1, v12
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v16, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 1, v13
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 1, v14
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v19, v18
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v8, v11, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v12, v15, vcc
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
-; GCN-NEXT:    v_ashrrev_i32_e32 v8, 31, v3
-; GCN-NEXT:    v_mul_hi_u32 v4, v18, v4
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v12, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v8
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v18, v4
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v10, v10, v22, s[4:5]
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v15
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v16
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v13, s[4:5]
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v15
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v16
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v10
+; GCN-NEXT:    v_ashrrev_i32_e32 v8, 31, v8
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v13, v16, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v8
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v9
+; GCN-NEXT:    v_mul_hi_u32 v4, v18, v19
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v20, s[4:5]
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v10, v5, vcc
-; GCN-NEXT:    v_mul_lo_u32 v5, v4, v7
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v17
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v17
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_xor_b32_e32 v6, v8, v14
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
-; GCN-NEXT:    v_sub_i32_e32 v8, vcc, v3, v7
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
-; GCN-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v8, vcc
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v6
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v14, v17, vcc
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
+; GCN-NEXT:    v_max_i32_e32 v5, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v18, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v5, v4
+; GCN-NEXT:    v_xor_b32_e32 v2, v2, v10
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GCN-NEXT:    v_mul_lo_u32 v6, v4, v11
+; GCN-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v5, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GCN-NEXT:    v_sub_i32_e32 v7, vcc, v5, v11
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v11
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v11
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GCN-NEXT:    v_xor_b32_e32 v4, v4, v3
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
@@ -950,119 +931,115 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; TONGA-NEXT:    s_mov_b32 s8, s0
 ; TONGA-NEXT:    s_mov_b32 s9, s1
 ; TONGA-NEXT:    s_waitcnt vmcnt(1)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; TONGA-NEXT:    v_sub_u32_e32 v12, vcc, 0, v1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
-; TONGA-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
-; TONGA-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v4, v9
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v5, v11
-; TONGA-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v8
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v1, v10
-; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v9
-; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v11
-; TONGA-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
-; TONGA-NEXT:    v_xor_b32_e32 v15, v8, v9
-; TONGA-NEXT:    v_xor_b32_e32 v16, v10, v11
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, v6, v13
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v8
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v10
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v8, v4
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v10, v5
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v2, v12
-; TONGA-NEXT:    v_xor_b32_e32 v6, v6, v13
-; TONGA-NEXT:    v_xor_b32_e32 v17, v12, v13
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v12
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v12, v6
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v10, v10
-; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, 0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v10, vcc, 0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v5
+; TONGA-NEXT:    v_sub_u32_e32 v16, vcc, 0, v6
+; TONGA-NEXT:    v_xor_b32_e32 v8, v0, v4
+; TONGA-NEXT:    v_xor_b32_e32 v11, v1, v5
+; TONGA-NEXT:    v_xor_b32_e32 v14, v2, v6
+; TONGA-NEXT:    v_max_i32_e32 v4, v4, v10
+; TONGA-NEXT:    v_max_i32_e32 v5, v5, v13
+; TONGA-NEXT:    v_max_i32_e32 v6, v6, v16
+; TONGA-NEXT:    v_max_i32_e32 v1, v1, v12
+; TONGA-NEXT:    v_ashrrev_i32_e32 v10, 31, v14
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v12, v4
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v14, v5
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v16, v6
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, 0, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v12, v12
-; TONGA-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
-; TONGA-NEXT:    v_mul_f32_e32 v10, 0x4f7ffffe, v10
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v14, v14
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v16, v16
+; TONGA-NEXT:    v_sub_u32_e32 v15, vcc, 0, v2
 ; TONGA-NEXT:    v_mul_f32_e32 v12, 0x4f7ffffe, v12
+; TONGA-NEXT:    v_mul_f32_e32 v14, 0x4f7ffffe, v14
+; TONGA-NEXT:    v_mul_f32_e32 v16, 0x4f7ffffe, v16
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v12, v12
-; TONGA-NEXT:    v_sub_u32_e32 v11, vcc, 0, v5
-; TONGA-NEXT:    v_mul_lo_u32 v9, v9, v8
-; TONGA-NEXT:    v_mul_lo_u32 v11, v11, v10
-; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v6
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v14, v14
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v16, v16
+; TONGA-NEXT:    v_sub_u32_e32 v17, vcc, 0, v7
+; TONGA-NEXT:    v_max_i32_e32 v0, v0, v9
+; TONGA-NEXT:    v_ashrrev_i32_e32 v9, 31, v11
+; TONGA-NEXT:    v_max_i32_e32 v2, v2, v15
+; TONGA-NEXT:    v_max_i32_e32 v11, v7, v17
+; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v4
+; TONGA-NEXT:    v_sub_u32_e32 v15, vcc, 0, v5
+; TONGA-NEXT:    v_sub_u32_e32 v17, vcc, 0, v6
 ; TONGA-NEXT:    v_mul_lo_u32 v13, v13, v12
-; TONGA-NEXT:    v_mul_hi_u32 v9, v8, v9
-; TONGA-NEXT:    v_mul_hi_u32 v11, v10, v11
-; TONGA-NEXT:    v_ashrrev_i32_e32 v14, 31, v7
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, v7, v14
+; TONGA-NEXT:    v_mul_lo_u32 v15, v15, v14
+; TONGA-NEXT:    v_mul_lo_u32 v17, v17, v16
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v18, v11
 ; TONGA-NEXT:    v_mul_hi_u32 v13, v12, v13
-; TONGA-NEXT:    v_xor_b32_e32 v7, v7, v14
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v18, v7
-; TONGA-NEXT:    v_add_u32_e32 v8, vcc, v8, v9
-; TONGA-NEXT:    v_add_u32_e32 v9, vcc, v10, v11
-; TONGA-NEXT:    v_mul_hi_u32 v8, v0, v8
-; TONGA-NEXT:    v_mul_hi_u32 v9, v1, v9
-; TONGA-NEXT:    v_add_u32_e32 v10, vcc, v12, v13
-; TONGA-NEXT:    v_mul_hi_u32 v10, v2, v10
+; TONGA-NEXT:    v_mul_hi_u32 v15, v14, v15
+; TONGA-NEXT:    v_mul_hi_u32 v17, v16, v17
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v18, v18
-; TONGA-NEXT:    v_mul_lo_u32 v11, v8, v4
-; TONGA-NEXT:    v_mul_lo_u32 v13, v9, v5
-; TONGA-NEXT:    v_mul_lo_u32 v21, v10, v6
+; TONGA-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
+; TONGA-NEXT:    v_add_u32_e32 v13, vcc, v14, v15
+; TONGA-NEXT:    v_add_u32_e32 v14, vcc, v16, v17
+; TONGA-NEXT:    v_mul_hi_u32 v12, v0, v12
+; TONGA-NEXT:    v_mul_hi_u32 v13, v1, v13
+; TONGA-NEXT:    v_mul_hi_u32 v14, v2, v14
 ; TONGA-NEXT:    v_mul_f32_e32 v18, 0x4f7ffffe, v18
+; TONGA-NEXT:    v_mul_lo_u32 v15, v12, v4
+; TONGA-NEXT:    v_mul_lo_u32 v17, v13, v5
+; TONGA-NEXT:    v_mul_lo_u32 v21, v14, v6
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v18, v18
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v11
-; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v13
-; TONGA-NEXT:    v_add_u32_e32 v12, vcc, 1, v8
-; TONGA-NEXT:    v_add_u32_e32 v20, vcc, 1, v9
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v15
+; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v17
+; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, v2, v21
+; TONGA-NEXT:    v_add_u32_e32 v16, vcc, 1, v12
+; TONGA-NEXT:    v_add_u32_e32 v20, vcc, 1, v13
+; TONGA-NEXT:    v_add_u32_e32 v15, vcc, 1, v14
 ; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v4
 ; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v1, v5
-; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, v2, v21
-; TONGA-NEXT:    v_sub_u32_e32 v11, vcc, v0, v4
-; TONGA-NEXT:    v_cndmask_b32_e64 v8, v8, v12, s[0:1]
-; TONGA-NEXT:    v_sub_u32_e32 v12, vcc, v1, v5
-; TONGA-NEXT:    v_cndmask_b32_e64 v9, v9, v20, s[2:3]
-; TONGA-NEXT:    v_sub_u32_e32 v19, vcc, 0, v7
-; TONGA-NEXT:    v_add_u32_e32 v22, vcc, 1, v10
-; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, v2, v6
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v11, s[0:1]
-; TONGA-NEXT:    v_add_u32_e32 v11, vcc, 1, v8
-; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v12, s[2:3]
-; TONGA-NEXT:    v_add_u32_e32 v12, vcc, 1, v9
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; TONGA-NEXT:    v_sub_u32_e32 v19, vcc, 0, v11
+; TONGA-NEXT:    v_sub_u32_e32 v17, vcc, v0, v4
+; TONGA-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s[0:1]
+; TONGA-NEXT:    v_sub_u32_e32 v16, vcc, v1, v5
+; TONGA-NEXT:    v_cndmask_b32_e64 v13, v13, v20, s[2:3]
+; TONGA-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[4:5]
+; TONGA-NEXT:    v_mul_lo_u32 v19, v19, v18
+; TONGA-NEXT:    v_sub_u32_e32 v20, vcc, v2, v6
+; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v17, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v15, vcc, 1, v12
+; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v16, s[2:3]
+; TONGA-NEXT:    v_add_u32_e32 v16, vcc, 1, v13
+; TONGA-NEXT:    v_add_u32_e32 v17, vcc, 1, v14
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; TONGA-NEXT:    v_mul_lo_u32 v4, v19, v18
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v8, v11, vcc
+; TONGA-NEXT:    v_cndmask_b32_e32 v0, v12, v15, vcc
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
-; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v3
-; TONGA-NEXT:    v_mul_hi_u32 v4, v18, v4
-; TONGA-NEXT:    v_cndmask_b32_e32 v1, v9, v12, vcc
-; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v3, v8
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v8
-; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v18, v4
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
-; TONGA-NEXT:    v_mul_hi_u32 v4, v3, v4
-; TONGA-NEXT:    v_cndmask_b32_e64 v10, v10, v22, s[4:5]
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v15
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v16
-; TONGA-NEXT:    v_cndmask_b32_e64 v2, v2, v13, s[4:5]
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v15
-; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v16
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v10
+; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v8
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v13, v16, vcc
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v8
+; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v9
+; TONGA-NEXT:    v_mul_hi_u32 v4, v18, v19
+; TONGA-NEXT:    v_cndmask_b32_e64 v2, v2, v20, s[4:5]
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v8
+; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v9
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
-; TONGA-NEXT:    v_cndmask_b32_e32 v2, v10, v5, vcc
-; TONGA-NEXT:    v_mul_lo_u32 v5, v4, v7
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v17
-; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, v2, v17
-; TONGA-NEXT:    v_sub_u32_e32 v3, vcc, v3, v5
-; TONGA-NEXT:    v_xor_b32_e32 v6, v8, v14
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v4
-; TONGA-NEXT:    v_sub_u32_e32 v8, vcc, v3, v7
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
-; TONGA-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; TONGA-NEXT:    v_cndmask_b32_e32 v3, v3, v8, vcc
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v4
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
-; TONGA-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v6
-; TONGA-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
+; TONGA-NEXT:    v_cndmask_b32_e32 v2, v14, v17, vcc
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, 0, v3
+; TONGA-NEXT:    v_max_i32_e32 v5, v3, v5
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v18, v4
+; TONGA-NEXT:    v_mul_hi_u32 v4, v5, v4
+; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v10
+; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, v2, v10
+; TONGA-NEXT:    v_mul_lo_u32 v6, v4, v11
+; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v7
+; TONGA-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, v5, v6
+; TONGA-NEXT:    v_add_u32_e32 v6, vcc, 1, v4
+; TONGA-NEXT:    v_sub_u32_e32 v7, vcc, v5, v11
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v11
+; TONGA-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; TONGA-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; TONGA-NEXT:    v_add_u32_e32 v6, vcc, 1, v4
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v11
+; TONGA-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v3
+; TONGA-NEXT:    v_sub_u32_e32 v3, vcc, v4, v3
 ; TONGA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
 ; TONGA-NEXT:    s_endpgm
 ;
@@ -1078,138 +1055,126 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX9-NEXT:    s_mov_b32 s9, s7
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
 ; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
-; GFX9-NEXT:    s_ashr_i32 s4, s1, 31
-; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_xor_b32 s6, s1, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_abs_i32 s1, s0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_readfirstlane_b32 s8, v4
-; GFX9-NEXT:    s_ashr_i32 s9, s8, 31
-; GFX9-NEXT:    s_add_i32 s8, s8, s9
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX9-NEXT:    s_xor_b32 s0, s7, s0
+; GFX9-NEXT:    s_ashr_i32 s8, s0, 31
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s9, s4
-; GFX9-NEXT:    s_xor_b32 s8, s8, s9
-; GFX9-NEXT:    s_sub_i32 s9, 0, s6
+; GFX9-NEXT:    s_sub_i32 s0, 0, s1
+; GFX9-NEXT:    s_abs_i32 s7, s7
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v1
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s10, v0
-; GFX9-NEXT:    s_mul_i32 s9, s9, s10
-; GFX9-NEXT:    s_mul_hi_u32 s9, s10, s9
-; GFX9-NEXT:    s_add_i32 s10, s10, s9
-; GFX9-NEXT:    s_mul_hi_u32 s9, s8, s10
-; GFX9-NEXT:    s_mul_i32 s10, s9, s6
-; GFX9-NEXT:    s_sub_i32 s8, s8, s10
-; GFX9-NEXT:    s_add_i32 s11, s9, 1
-; GFX9-NEXT:    s_sub_i32 s10, s8, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s8, s6
-; GFX9-NEXT:    s_cselect_b32 s9, s11, s9
-; GFX9-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX9-NEXT:    s_add_i32 s10, s9, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s8, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s10, s9
-; GFX9-NEXT:    s_ashr_i32 s8, s7, 31
-; GFX9-NEXT:    s_add_i32 s7, s7, s8
-; GFX9-NEXT:    s_xor_b32 s7, s7, s8
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
+; GFX9-NEXT:    s_mul_i32 s0, s0, s9
+; GFX9-NEXT:    s_mul_hi_u32 s0, s9, s0
+; GFX9-NEXT:    s_add_i32 s9, s9, s0
+; GFX9-NEXT:    s_mul_hi_u32 s0, s7, s9
+; GFX9-NEXT:    s_mul_i32 s9, s0, s1
+; GFX9-NEXT:    s_sub_i32 s7, s7, s9
+; GFX9-NEXT:    s_add_i32 s10, s0, 1
+; GFX9-NEXT:    s_sub_i32 s9, s7, s1
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s1
+; GFX9-NEXT:    s_cselect_b32 s0, s10, s0
+; GFX9-NEXT:    s_cselect_b32 s7, s9, s7
+; GFX9-NEXT:    s_add_i32 s9, s0, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s9, s0
+; GFX9-NEXT:    s_abs_i32 s7, s6
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT:    v_readfirstlane_b32 s10, v5
-; GFX9-NEXT:    s_ashr_i32 s11, s10, 31
-; GFX9-NEXT:    s_xor_b32 s6, s6, s4
+; GFX9-NEXT:    s_xor_b32 s1, s1, s8
+; GFX9-NEXT:    s_sub_i32 s10, 0, s7
+; GFX9-NEXT:    s_sub_i32 s8, s1, s8
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_add_i32 s10, s10, s11
-; GFX9-NEXT:    s_xor_b32 s8, s11, s8
-; GFX9-NEXT:    s_sub_i32 s4, s6, s4
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v5
+; GFX9-NEXT:    s_xor_b32 s6, s9, s6
+; GFX9-NEXT:    s_abs_i32 s9, s9
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s6, s10, s11
-; GFX9-NEXT:    s_sub_i32 s10, 0, s7
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s11, v0
-; GFX9-NEXT:    s_mul_i32 s10, s10, s11
-; GFX9-NEXT:    s_mul_hi_u32 s10, s11, s10
-; GFX9-NEXT:    s_add_i32 s11, s11, s10
-; GFX9-NEXT:    s_mul_hi_u32 s10, s6, s11
-; GFX9-NEXT:    s_mul_i32 s11, s10, s7
-; GFX9-NEXT:    s_sub_i32 s6, s6, s11
-; GFX9-NEXT:    s_add_i32 s12, s10, 1
-; GFX9-NEXT:    s_sub_i32 s11, s6, s7
-; GFX9-NEXT:    s_cmp_ge_u32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s10, s12, s10
-; GFX9-NEXT:    s_cselect_b32 s6, s11, s6
-; GFX9-NEXT:    s_add_i32 s11, s10, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s6, s11, s10
-; GFX9-NEXT:    s_ashr_i32 s7, s9, 31
-; GFX9-NEXT:    s_add_i32 s9, s9, s7
-; GFX9-NEXT:    s_xor_b32 s9, s9, s7
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 31
+; GFX9-NEXT:    s_mov_b32 s0, s4
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-NEXT:    s_mul_i32 s10, s10, s1
+; GFX9-NEXT:    s_mul_hi_u32 s10, s1, s10
+; GFX9-NEXT:    s_add_i32 s1, s1, s10
+; GFX9-NEXT:    s_mul_hi_u32 s1, s9, s1
+; GFX9-NEXT:    s_mul_i32 s10, s1, s7
+; GFX9-NEXT:    s_sub_i32 s9, s9, s10
+; GFX9-NEXT:    s_add_i32 s11, s1, 1
+; GFX9-NEXT:    s_sub_i32 s10, s9, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s7
+; GFX9-NEXT:    s_cselect_b32 s1, s11, s1
+; GFX9-NEXT:    s_cselect_b32 s9, s10, s9
+; GFX9-NEXT:    s_add_i32 s10, s1, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s7
+; GFX9-NEXT:    s_cselect_b32 s7, s10, s1
+; GFX9-NEXT:    s_abs_i32 s9, s4
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT:    v_readfirstlane_b32 s11, v6
-; GFX9-NEXT:    s_ashr_i32 s12, s11, 31
-; GFX9-NEXT:    s_xor_b32 s6, s6, s8
+; GFX9-NEXT:    s_xor_b32 s7, s7, s6
+; GFX9-NEXT:    s_sub_i32 s11, 0, s9
+; GFX9-NEXT:    s_sub_i32 s6, s7, s6
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_add_i32 s11, s11, s12
-; GFX9-NEXT:    s_xor_b32 s7, s12, s7
-; GFX9-NEXT:    s_sub_i32 s6, s6, s8
+; GFX9-NEXT:    v_readfirstlane_b32 s10, v6
+; GFX9-NEXT:    s_xor_b32 s4, s10, s4
+; GFX9-NEXT:    s_abs_i32 s10, s10
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s8, s11, s12
-; GFX9-NEXT:    s_sub_i32 s11, 0, s9
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 31
+; GFX9-NEXT:    s_mov_b32 s1, s5
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v0
+; GFX9-NEXT:    s_mul_i32 s11, s11, s7
+; GFX9-NEXT:    s_mul_hi_u32 s11, s7, s11
+; GFX9-NEXT:    s_add_i32 s7, s7, s11
+; GFX9-NEXT:    s_mul_hi_u32 s7, s10, s7
+; GFX9-NEXT:    s_mul_i32 s11, s7, s9
+; GFX9-NEXT:    s_sub_i32 s10, s10, s11
+; GFX9-NEXT:    s_add_i32 s12, s7, 1
+; GFX9-NEXT:    s_sub_i32 s11, s10, s9
+; GFX9-NEXT:    s_cmp_ge_u32 s10, s9
+; GFX9-NEXT:    s_cselect_b32 s7, s12, s7
+; GFX9-NEXT:    s_cselect_b32 s10, s11, s10
+; GFX9-NEXT:    s_add_i32 s11, s7, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s10, s9
+; GFX9-NEXT:    s_cselect_b32 s7, s11, s7
+; GFX9-NEXT:    s_abs_i32 s9, s5
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s9
+; GFX9-NEXT:    s_xor_b32 s7, s7, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s8
+; GFX9-NEXT:    s_sub_i32 s8, 0, s9
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX9-NEXT:    s_sub_i32 s4, s7, s4
 ; GFX9-NEXT:    v_readfirstlane_b32 s10, v7
-; GFX9-NEXT:    v_readfirstlane_b32 s12, v0
-; GFX9-NEXT:    s_mul_i32 s11, s11, s12
-; GFX9-NEXT:    s_mul_hi_u32 s11, s12, s11
-; GFX9-NEXT:    s_add_i32 s12, s12, s11
-; GFX9-NEXT:    s_mul_hi_u32 s11, s8, s12
-; GFX9-NEXT:    s_mul_i32 s12, s11, s9
-; GFX9-NEXT:    s_sub_i32 s8, s8, s12
-; GFX9-NEXT:    s_add_i32 s13, s11, 1
-; GFX9-NEXT:    s_sub_i32 s12, s8, s9
-; GFX9-NEXT:    s_cmp_ge_u32 s8, s9
-; GFX9-NEXT:    s_cselect_b32 s11, s13, s11
-; GFX9-NEXT:    s_cselect_b32 s8, s12, s8
-; GFX9-NEXT:    s_add_i32 s12, s11, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s8, s9
-; GFX9-NEXT:    s_cselect_b32 s8, s12, s11
-; GFX9-NEXT:    s_ashr_i32 s9, s5, 31
-; GFX9-NEXT:    s_add_i32 s5, s5, s9
-; GFX9-NEXT:    s_xor_b32 s5, s5, s9
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s5
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    s_ashr_i32 s4, s10, 31
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    s_xor_b32 s6, s8, s7
-; GFX9-NEXT:    s_xor_b32 s8, s4, s9
-; GFX9-NEXT:    s_sub_i32 s6, s6, s7
 ; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    s_sub_i32 s7, 0, s5
-; GFX9-NEXT:    s_add_i32 s10, s10, s4
-; GFX9-NEXT:    s_xor_b32 s4, s10, s4
-; GFX9-NEXT:    v_readfirstlane_b32 s9, v2
-; GFX9-NEXT:    s_mul_i32 s7, s7, s9
-; GFX9-NEXT:    s_mul_hi_u32 s7, s9, s7
-; GFX9-NEXT:    s_add_i32 s9, s9, s7
-; GFX9-NEXT:    s_mul_hi_u32 s7, s4, s9
-; GFX9-NEXT:    s_mul_i32 s9, s7, s5
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
+; GFX9-NEXT:    s_abs_i32 s6, s10
+; GFX9-NEXT:    s_xor_b32 s5, s10, s5
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 31
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v2
+; GFX9-NEXT:    s_mul_i32 s8, s8, s7
+; GFX9-NEXT:    s_mul_hi_u32 s8, s7, s8
+; GFX9-NEXT:    s_add_i32 s7, s7, s8
+; GFX9-NEXT:    s_mul_hi_u32 s7, s6, s7
+; GFX9-NEXT:    s_mul_i32 s8, s7, s9
+; GFX9-NEXT:    s_sub_i32 s6, s6, s8
 ; GFX9-NEXT:    s_add_i32 s10, s7, 1
-; GFX9-NEXT:    s_sub_i32 s9, s4, s5
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s5
+; GFX9-NEXT:    s_sub_i32 s8, s6, s9
+; GFX9-NEXT:    s_cmp_ge_u32 s6, s9
 ; GFX9-NEXT:    s_cselect_b32 s7, s10, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX9-NEXT:    s_add_i32 s9, s7, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s7
-; GFX9-NEXT:    s_xor_b32 s4, s4, s8
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX9-NEXT:    s_add_i32 s8, s7, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s6, s9
+; GFX9-NEXT:    s_cselect_b32 s6, s8, s7
+; GFX9-NEXT:    s_xor_b32 s6, s6, s5
+; GFX9-NEXT:    s_sub_i32 s5, s6, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -2009,20 +1974,19 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_bfe_i32 v2, v1, 0, 25
-; GCN-NEXT:    v_bfe_i32 v1, v1, 24, 1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v1
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v1
+; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 25
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
+; GCN-NEXT:    v_max_i32_e32 v2, v1, v2
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v2
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_bfe_i32 v5, v0, 0, 25
+; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GCN-NEXT:    v_bfe_i32 v0, v0, 24, 1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v0
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v0
+; GCN-NEXT:    v_max_i32_e32 v5, v0, v5
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_xor_b32_e32 v5, v5, v0
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
@@ -2030,7 +1994,7 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-NEXT:    v_mul_lo_u32 v1, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v5, v1
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v1, v2
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, v2, v1
 ; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -2057,20 +2021,19 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; TONGA-NEXT:    s_mov_b32 s0, s4
 ; TONGA-NEXT:    s_mov_b32 s1, s5
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_bfe_i32 v2, v1, 0, 25
-; TONGA-NEXT:    v_bfe_i32 v1, v1, 24, 1
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v1
+; TONGA-NEXT:    v_bfe_i32 v1, v1, 0, 25
+; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, 0, v1
+; TONGA-NEXT:    v_max_i32_e32 v2, v1, v2
 ; TONGA-NEXT:    v_cvt_f32_u32_e32 v3, v2
 ; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v2
-; TONGA-NEXT:    v_bfe_i32 v5, v0, 0, 25
+; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; TONGA-NEXT:    v_bfe_i32 v0, v0, 24, 1
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v5, v0
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, 0, v0
+; TONGA-NEXT:    v_max_i32_e32 v5, v0, v5
 ; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v0
 ; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
+; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
 ; TONGA-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; TONGA-NEXT:    v_mul_hi_u32 v4, v3, v4
 ; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
@@ -2078,7 +2041,7 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; TONGA-NEXT:    v_mul_lo_u32 v1, v3, v2
 ; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
 ; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v5, v1
-; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, v1, v2
+; TONGA-NEXT:    v_subrev_u32_e32 v5, vcc, v2, v1
 ; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
 ; TONGA-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; TONGA-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -2102,42 +2065,39 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX9-NEXT:    s_mov_b32 s8, s6
 ; GFX9-NEXT:    s_mov_b32 s9, s7
 ; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX9-NEXT:    s_bfe_i32 s1, s0, 0x190000
-; GFX9-NEXT:    s_bfe_i32 s6, s0, 0x10018
-; GFX9-NEXT:    s_add_i32 s1, s1, s6
-; GFX9-NEXT:    s_xor_b32 s7, s1, s6
+; GFX9-NEXT:    s_bfe_i32 s6, s0, 0x190000
+; GFX9-NEXT:    s_abs_i32 s7, s6
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s7
 ; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v1
-; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x190000
-; GFX9-NEXT:    s_bfe_i32 s4, s4, 0x10018
-; GFX9-NEXT:    s_add_i32 s5, s5, s4
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    s_bfe_i32 s4, s4, 0x190000
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX9-NEXT:    s_xor_b32 s5, s4, s6
+; GFX9-NEXT:    s_sub_i32 s6, 0, s7
+; GFX9-NEXT:    s_abs_i32 s4, s4
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s6, s4, s6
-; GFX9-NEXT:    s_xor_b32 s4, s5, s4
-; GFX9-NEXT:    s_sub_i32 s5, 0, s7
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 31
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
-; GFX9-NEXT:    s_mul_i32 s5, s5, s8
-; GFX9-NEXT:    s_mul_hi_u32 s5, s8, s5
-; GFX9-NEXT:    s_add_i32 s8, s8, s5
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, s8
-; GFX9-NEXT:    s_mul_i32 s8, s5, s7
+; GFX9-NEXT:    s_mul_i32 s6, s6, s8
+; GFX9-NEXT:    s_mul_hi_u32 s6, s8, s6
+; GFX9-NEXT:    s_add_i32 s8, s8, s6
+; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s8
+; GFX9-NEXT:    s_mul_i32 s8, s6, s7
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_add_i32 s9, s5, 1
+; GFX9-NEXT:    s_add_i32 s9, s6, 1
 ; GFX9-NEXT:    s_sub_i32 s8, s4, s7
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX9-NEXT:    s_cselect_b32 s6, s9, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_add_i32 s8, s5, 1
+; GFX9-NEXT:    s_add_i32 s8, s6, 1
 ; GFX9-NEXT:    s_cmp_ge_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s5
-; GFX9-NEXT:    s_xor_b32 s4, s4, s6
-; GFX9-NEXT:    s_sub_i32 s4, s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s6
+; GFX9-NEXT:    s_xor_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
 ; GFX9-NEXT:    s_bfe_i32 s4, s4, 0x190000
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index b086640c72f80471c3c1fdf06ea11ae106087253..c310e257adadc6a7c5e2cdfbc4bda8ad35fbbd50 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -147,11 +147,11 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s2, s9, 31
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[6:7]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[6:7], s[0:1]
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
 ; GCN-IR-NEXT:    s_sub_u32 s12, s6, s0
 ; GCN-IR-NEXT:    s_subb_u32 s13, s7, s0
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[8:9]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[2:3]
 ; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
 ; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[12:13], 0
@@ -357,13 +357,13 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v12
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v13, 31, v3
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v12
 ; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v0, v12
 ; GCN-IR-NEXT:    v_subb_u32_e32 v7, vcc, v1, v12, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v13, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v13, v3
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v2, v13
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v3, v13
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v13
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
@@ -587,43 +587,39 @@ define i64 @v_test_sdiv24_64(i64 %x, i64 %y) {
 define amdgpu_kernel void @s_test_sdiv32_64(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_sdiv32_64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dword s2, s[0:1], 0xe
+; GCN-NEXT:    s_load_dword s8, s[0:1], 0xe
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s8
-; GCN-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-NEXT:    s_abs_i32 s9, s8
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s9
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_abs_i32 s0, s3
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GCN-NEXT:    s_mov_b32 s5, s1
-; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_xor_b32 s1, s3, s8
+; GCN-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GCN-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-NEXT:    s_add_i32 s3, s3, s2
-; GCN-NEXT:    s_xor_b32 s3, s3, s2
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT:    s_xor_b32 s0, s2, s8
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GCN-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-NEXT:    s_mul_i32 s2, s1, s9
-; GCN-NEXT:    s_sub_i32 s2, s3, s2
-; GCN-NEXT:    s_add_i32 s8, s1, 1
-; GCN-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-NEXT:    s_add_i32 s3, s1, 1
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    s_add_i32 s8, s2, 1
+; GCN-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-NEXT:    s_add_i32 s3, s2, 1
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -632,43 +628,39 @@ define amdgpu_kernel void @s_test_sdiv32_64(ptr addrspace(1) %out, i64 %x, i64 %
 ;
 ; GCN-IR-LABEL: s_test_sdiv32_64:
 ; GCN-IR:       ; %bb.0:
-; GCN-IR-NEXT:    s_load_dword s2, s[0:1], 0xe
+; GCN-IR-NEXT:    s_load_dword s8, s[0:1], 0xe
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s8
-; GCN-IR-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-IR-NEXT:    s_abs_i32 s9, s8
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s9
+; GCN-IR-NEXT:    s_mov_b32 s4, s0
+; GCN-IR-NEXT:    s_abs_i32 s0, s3
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GCN-IR-NEXT:    s_mov_b32 s5, s1
-; GCN-IR-NEXT:    s_mov_b32 s4, s0
+; GCN-IR-NEXT:    s_xor_b32 s1, s3, s8
+; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-IR-NEXT:    s_add_i32 s3, s3, s2
-; GCN-IR-NEXT:    s_xor_b32 s3, s3, s2
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-IR-NEXT:    s_xor_b32 s0, s2, s8
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-IR-NEXT:    s_mul_i32 s2, s1, s9
-; GCN-IR-NEXT:    s_sub_i32 s2, s3, s2
-; GCN-IR-NEXT:    s_add_i32 s8, s1, 1
-; GCN-IR-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-IR-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-IR-NEXT:    s_add_i32 s3, s1, 1
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-IR-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-IR-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-IR-NEXT:    s_add_i32 s8, s2, 1
+; GCN-IR-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-IR-NEXT:    s_add_i32 s3, s2, 1
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-IR-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -688,41 +680,38 @@ define amdgpu_kernel void @s_test_sdiv31_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s8
-; GCN-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-NEXT:    s_ashr_i64 s[8:9], s[2:3], 33
+; GCN-NEXT:    s_abs_i32 s9, s8
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s9
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_abs_i32 s0, s2
+; GCN-NEXT:    s_xor_b32 s1, s2, s8
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s0, s3, s8
+; GCN-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-NEXT:    s_mul_i32 s3, s1, s9
-; GCN-NEXT:    s_sub_i32 s2, s2, s3
-; GCN-NEXT:    s_add_i32 s8, s1, 1
-; GCN-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-NEXT:    s_add_i32 s3, s1, 1
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    s_add_i32 s8, s2, 1
+; GCN-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-NEXT:    s_add_i32 s3, s2, 1
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -735,41 +724,38 @@ define amdgpu_kernel void @s_test_sdiv31_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-IR-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s8
-; GCN-IR-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 33
+; GCN-IR-NEXT:    s_abs_i32 s9, s8
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s9
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s9
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    s_mov_b32 s4, s0
+; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s0, s2
+; GCN-IR-NEXT:    s_xor_b32 s1, s2, s8
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-IR-NEXT:    s_xor_b32 s2, s2, s3
-; GCN-IR-NEXT:    s_xor_b32 s0, s3, s8
+; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-IR-NEXT:    s_mul_i32 s3, s1, s9
-; GCN-IR-NEXT:    s_sub_i32 s2, s2, s3
-; GCN-IR-NEXT:    s_add_i32 s8, s1, 1
-; GCN-IR-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-IR-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-IR-NEXT:    s_add_i32 s3, s1, 1
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-IR-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-IR-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-IR-NEXT:    s_add_i32 s8, s2, 1
+; GCN-IR-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-IR-NEXT:    s_add_i32 s3, s2, 1
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-IR-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -856,41 +842,38 @@ define amdgpu_kernel void @s_test_sdiv25_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s8
-; GCN-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-NEXT:    s_ashr_i64 s[8:9], s[2:3], 39
+; GCN-NEXT:    s_abs_i32 s9, s8
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s9
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_abs_i32 s0, s2
+; GCN-NEXT:    s_xor_b32 s1, s2, s8
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s0, s3, s8
+; GCN-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-NEXT:    s_mul_i32 s3, s1, s9
-; GCN-NEXT:    s_sub_i32 s2, s2, s3
-; GCN-NEXT:    s_add_i32 s8, s1, 1
-; GCN-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-NEXT:    s_add_i32 s3, s1, 1
-; GCN-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    s_add_i32 s8, s2, 1
+; GCN-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-NEXT:    s_add_i32 s3, s2, 1
+; GCN-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -903,41 +886,38 @@ define amdgpu_kernel void @s_test_sdiv25_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-IR-NEXT:    s_ashr_i32 s8, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s8
-; GCN-IR-NEXT:    s_xor_b32 s9, s2, s8
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 39
+; GCN-IR-NEXT:    s_abs_i32 s9, s8
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s9
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s9
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    s_mov_b32 s4, s0
+; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s0, s2
+; GCN-IR-NEXT:    s_xor_b32 s1, s2, s8
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-IR-NEXT:    s_xor_b32 s2, s2, s3
-; GCN-IR-NEXT:    s_xor_b32 s0, s3, s8
+; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 31
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
-; GCN-IR-NEXT:    s_mul_i32 s3, s1, s9
-; GCN-IR-NEXT:    s_sub_i32 s2, s2, s3
-; GCN-IR-NEXT:    s_add_i32 s8, s1, 1
-; GCN-IR-NEXT:    s_sub_i32 s3, s2, s9
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s8, s1
-; GCN-IR-NEXT:    s_cselect_b32 s2, s3, s2
-; GCN-IR-NEXT:    s_add_i32 s3, s1, 1
-; GCN-IR-NEXT:    s_cmp_ge_u32 s2, s9
-; GCN-IR-NEXT:    s_cselect_b32 s1, s3, s1
-; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-IR-NEXT:    v_readfirstlane_b32 s2, v0
+; GCN-IR-NEXT:    s_mul_i32 s3, s2, s9
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-IR-NEXT:    s_add_i32 s8, s2, 1
+; GCN-IR-NEXT:    s_sub_i32 s3, s0, s9
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s2, s8, s2
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-IR-NEXT:    s_add_i32 s3, s2, 1
+; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s9
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, s2
+; GCN-IR-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-IR-NEXT:    s_sub_i32 s0, s0, s1
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -1100,11 +1080,11 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[4:5], 16
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s5, 31
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[6:7]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[6:7], s[2:3]
 ; GCN-IR-NEXT:    s_mov_b32 s5, s4
 ; GCN-IR-NEXT:    s_sub_u32 s12, s6, s2
 ; GCN-IR-NEXT:    s_subb_u32 s13, s7, s2
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[4:5], s[8:9]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[4:5]
 ; GCN-IR-NEXT:    s_sub_u32 s6, s6, s4
 ; GCN-IR-NEXT:    s_subb_u32 s7, s7, s4
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
@@ -1310,7 +1290,7 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s3, 31
 ; GCN-IR-NEXT:    s_mov_b32 s5, s4
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
 ; GCN-IR-NEXT:    s_sub_u32 s2, s2, s4
 ; GCN-IR-NEXT:    s_subb_u32 s3, s3, s4
 ; GCN-IR-NEXT:    s_flbit_i32_b64 s14, s[2:3]
@@ -1493,8 +1473,8 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v12
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
@@ -1686,8 +1666,8 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v12
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
@@ -1788,8 +1768,8 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v10, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v10, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v10
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v10
 ; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, v0, v10
 ; GCN-IR-NEXT:    v_subb_u32_e32 v5, vcc, v1, v10, vcc
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll b/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ba04cdb795ce316fc1e8c1222bb6997931c7614d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=hawaii %s -o - | FileCheck %s
+
+; For all these tests we disable optimizations through function attributes
+; because the code we are exercising here needs phis and we want to keep the
+; IR small.
+
+; This code used to crash in SDISel because f16 was promoted to f32 through
+; a `f32 = vector_extract_elt <1 x f16>, i32 0`, which is illegal.
+; The invalid SDNode and thus, the crash was only exposed by the constant
+; folding.
+define void @phi_vec1half_to_f32_with_const_folding(ptr addrspace(1) %dst) #0 {
+; CHECK-LABEL: phi_vec1half_to_f32_with_const_folding:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  ; %bb.1: ; %bb
+; CHECK-NEXT:    v_cvt_f16_f32_e64 v2, s4
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_store_short v2, v[0:1], s[4:7], 0 addr64 offset:2
+; CHECK-NEXT:    v_cvt_f16_f32_e64 v2, s4
+; CHECK-NEXT:    buffer_store_short v2, v[0:1], s[4:7], 0 addr64
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %bb
+
+bb:
+  %phi = phi <1 x half> [ zeroinitializer, %entry ]
+  %res = shufflevector <1 x half> poison, <1 x half> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x half> %res, ptr addrspace(1) %dst
+  ret void
+}
+
+; Same as phi_vec1half_to_f32_with_const_folding but without the folding.
+; This test exercises the same invalid SDNode, but it happened to work by
+; accident before. Here we make sure the fix also work as expected in the
+; non-constant folding case.
+define void @phi_vec1half_to_f32(ptr addrspace(1) %src, ptr addrspace(1) %dst) #0 {
+; CHECK-LABEL: phi_vec1half_to_f32:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_cvt_f32_f16_e64 v0, v0
+; CHECK-NEXT:  ; %bb.1: ; %bb
+; CHECK-NEXT:    v_cvt_f16_f32_e64 v0, v0
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_store_short v0, v[2:3], s[4:7], 0 addr64 offset:2
+; CHECK-NEXT:    v_cvt_f16_f32_e64 v0, s4
+; CHECK-NEXT:    buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %input = load <1 x half>, ptr addrspace(1) %src
+  br label %bb
+
+bb:
+  %phi = phi <1 x half> [ %input, %entry ]
+  %res = shufflevector <1 x half> poison, <1 x half> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x half> %res, ptr addrspace(1) %dst
+  ret void
+}
+
+; Same as phi_vec1bf16_to_f32 but with bfloat instead of half.
+define void @phi_vec1bf16_to_f32(ptr addrspace(1) %src, ptr addrspace(1) %dst) #0 {
+; CHECK-LABEL: phi_vec1bf16_to_f32:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
+; CHECK-NEXT:    s_mov_b32 s4, 16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_lshlrev_b32_e64 v0, s4, v0
+; CHECK-NEXT:  ; %bb.1: ; %bb
+; CHECK-NEXT:    v_mul_f32_e64 v0, 1.0, v0
+; CHECK-NEXT:    s_mov_b32 s4, 16
+; CHECK-NEXT:    v_lshrrev_b32_e64 v0, s4, v0
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_store_short v0, v[2:3], s[4:7], 0 addr64 offset:2
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %input = load <1 x bfloat>, ptr addrspace(1) %src
+  br label %bb
+
+bb:
+  %phi = phi <1 x bfloat> [ %input, %entry ]
+  %res = shufflevector <1 x bfloat> poison, <1 x bfloat> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x bfloat> %res, ptr addrspace(1) %dst
+  ret void
+}
+
+; Same as phi_vec1half_to_f32_with_const_folding but with bfloat instead of half.
+define void @phi_vec1bf16_to_f32_with_const_folding(ptr addrspace(1) %dst) #0 {
+; CHECK-LABEL: phi_vec1bf16_to_f32_with_const_folding:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:  ; %bb.1: ; %bb
+; CHECK-NEXT:    v_mul_f32_e64 v2, 1.0, s4
+; CHECK-NEXT:    s_mov_b32 s4, 16
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, s4, v2
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s4, s6
+; CHECK-NEXT:    s_mov_b32 s5, s6
+; CHECK-NEXT:    buffer_store_short v2, v[0:1], s[4:7], 0 addr64 offset:2
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %bb
+
+bb:
+  %phi = phi <1 x bfloat> [ zeroinitializer, %entry ]
+  %res = shufflevector <1 x bfloat> poison, <1 x bfloat> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x bfloat> %res, ptr addrspace(1) %dst
+  ret void
+}
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir
index 981a853cb097c50bf4d31c7481f5d01ba51fec8c..8e6da4bf92ee03aaa9c13ff3716e8a6c0f1eae27 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir
@@ -1,7 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GFX908-SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -passes=regallocfast -o - %s | FileCheck -check-prefix=GFX908-SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regallocfast,prologepilog -o - %s | FileCheck -check-prefix=GFX908-EXPANDED %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GFX90A-SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -passes=regallocfast -o - %s | FileCheck -check-prefix=GFX90A-SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regallocfast,prologepilog -o - %s | FileCheck -check-prefix=GFX90A-EXPANDED %s
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/spill192.mir b/llvm/test/CodeGen/AMDGPU/spill192.mir
index 56882cd83106edc2c9705243361f6efa0395cc3d..42c792b1538ae717d6115d7d6d3d10ef709f9e26 100644
--- a/llvm/test/CodeGen/AMDGPU/spill192.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill192.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 192 bit registers works. We have to
diff --git a/llvm/test/CodeGen/AMDGPU/spill224.mir b/llvm/test/CodeGen/AMDGPU/spill224.mir
index 4525fc1cb70e6f92f4e211cb9d9b442813dd4433..5e53f93df95f7431240c598340e9bab005bb38fc 100644
--- a/llvm/test/CodeGen/AMDGPU/spill224.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill224.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 224 bit registers works.
diff --git a/llvm/test/CodeGen/AMDGPU/spill288.mir b/llvm/test/CodeGen/AMDGPU/spill288.mir
index 173056e6b9132651c9e804a53f5ac789613496de..3d5404a9c1ad5bc1999dae180f5d1e611f07d55c 100644
--- a/llvm/test/CodeGen/AMDGPU/spill288.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill288.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 288 bit registers works.
diff --git a/llvm/test/CodeGen/AMDGPU/spill320.mir b/llvm/test/CodeGen/AMDGPU/spill320.mir
index 828f524bb0677e03524f10a08386e2a69458dd64..4473a4d6648efe09513f06a848cee47b11a6c022 100644
--- a/llvm/test/CodeGen/AMDGPU/spill320.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill320.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 320 bit registers works.
diff --git a/llvm/test/CodeGen/AMDGPU/spill352.mir b/llvm/test/CodeGen/AMDGPU/spill352.mir
index ef620fae5b104ccd56b81e265ee3e8f579ce7b0a..8fa053a908b60eec30d46962d74cc304bc91192f 100644
--- a/llvm/test/CodeGen/AMDGPU/spill352.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill352.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 352 bit registers works.
diff --git a/llvm/test/CodeGen/AMDGPU/spill384.mir b/llvm/test/CodeGen/AMDGPU/spill384.mir
index 37cf5d0fc83cfbd51afbddc1d3e6406dbd40af25..cd604e4483b9c734beba0d414b0657e2dc866be8 100644
--- a/llvm/test/CodeGen/AMDGPU/spill384.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill384.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s
 # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s
 
 # Make sure spill/restore of 384 bit registers works.
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index ed7f27b367fdaff4d39a24c7a03d3d792f29a2f4..93fab7dff253bc5f96c2205c528123793856ba1b 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -335,11 +335,11 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
 ; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v14
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v14
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v14
-; GCN-IR-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v14, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v2, v2, v4
 ; GCN-IR-NEXT:    v_xor_b32_e32 v3, v3, v4
 ; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
 ; GCN-IR-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
@@ -655,37 +655,34 @@ define amdgpu_kernel void @s_test_srem25_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-NEXT:    s_abs_i32 s8, s2
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_abs_i32 s3, s2
+; GCN-NEXT:    s_ashr_i32 s0, s2, 31
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-NEXT:    s_sub_i32 s0, s2, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_xor_b32 s0, s0, s3
-; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GCN-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-NEXT:    s_sub_i32 s1, s3, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -699,37 +696,34 @@ define amdgpu_kernel void @s_test_srem25_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
-; GCN-IR-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s8, s2
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s8
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-IR-NEXT:    s_mov_b32 s4, s0
 ; GCN-IR-NEXT:    s_mov_b32 s5, s1
+; GCN-IR-NEXT:    s_mov_b32 s4, s0
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 39
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s3, s2
+; GCN-IR-NEXT:    s_ashr_i32 s0, s2, 31
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-IR-NEXT:    s_xor_b32 s2, s2, s3
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-IR-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-IR-NEXT:    s_sub_i32 s0, s2, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_xor_b32 s0, s0, s3
-; GCN-IR-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-IR-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-IR-NEXT:    s_sub_i32 s1, s3, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -750,37 +744,34 @@ define amdgpu_kernel void @s_test_srem31_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-NEXT:    s_abs_i32 s8, s2
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_abs_i32 s3, s2
+; GCN-NEXT:    s_ashr_i32 s0, s2, 31
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-NEXT:    s_sub_i32 s0, s2, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_xor_b32 s0, s0, s3
-; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GCN-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-NEXT:    s_sub_i32 s1, s3, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -794,37 +785,34 @@ define amdgpu_kernel void @s_test_srem31_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
-; GCN-IR-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s8, s2
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s8
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-IR-NEXT:    s_mov_b32 s4, s0
 ; GCN-IR-NEXT:    s_mov_b32 s5, s1
+; GCN-IR-NEXT:    s_mov_b32 s4, s0
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 33
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s3, s2
+; GCN-IR-NEXT:    s_ashr_i32 s0, s2, 31
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GCN-IR-NEXT:    s_xor_b32 s2, s2, s3
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-IR-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-IR-NEXT:    s_sub_i32 s0, s2, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_xor_b32 s0, s0, s3
-; GCN-IR-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-IR-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-IR-NEXT:    s_sub_i32 s1, s3, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -845,36 +833,33 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-NEXT:    s_abs_i32 s8, s2
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s4, s0
-; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GCN-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-NEXT:    s_add_i32 s3, s3, s2
-; GCN-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_abs_i32 s2, s3
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GCN-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-NEXT:    s_sub_i32 s0, s3, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-NEXT:    s_xor_b32 s0, s0, s2
-; GCN-NEXT:    s_sub_i32 s0, s0, s2
+; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-NEXT:    s_sub_i32 s1, s2, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -887,36 +872,33 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-IR-NEXT:    s_add_i32 s2, s2, s3
-; GCN-IR-NEXT:    s_xor_b32 s8, s2, s3
+; GCN-IR-NEXT:    s_abs_i32 s8, s2
 ; GCN-IR-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sub_i32 s2, 0, s8
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-IR-NEXT:    s_mov_b32 s4, s0
-; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-IR-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-IR-NEXT:    s_add_i32 s3, s3, s2
-; GCN-IR-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT:    s_abs_i32 s2, s3
+; GCN-IR-NEXT:    s_mov_b32 s5, s1
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GCN-IR-NEXT:    v_readfirstlane_b32 s0, v0
-; GCN-IR-NEXT:    s_mul_i32 s0, s0, s8
-; GCN-IR-NEXT:    s_sub_i32 s0, s3, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_sub_i32 s1, s0, s8
-; GCN-IR-NEXT:    s_cmp_ge_u32 s0, s8
-; GCN-IR-NEXT:    s_cselect_b32 s0, s1, s0
-; GCN-IR-NEXT:    s_xor_b32 s0, s0, s2
-; GCN-IR-NEXT:    s_sub_i32 s0, s0, s2
+; GCN-IR-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GCN-IR-NEXT:    s_mov_b32 s4, s0
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-IR-NEXT:    s_mul_i32 s1, s1, s8
+; GCN-IR-NEXT:    s_sub_i32 s1, s2, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_sub_i32 s2, s1, s8
+; GCN-IR-NEXT:    s_cmp_ge_u32 s1, s8
+; GCN-IR-NEXT:    s_cselect_b32 s1, s2, s1
+; GCN-IR-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-IR-NEXT:    s_sub_i32 s0, s1, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
@@ -1074,19 +1056,19 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[0:1], 31
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
-; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_mov_b32 s11, s10
 ; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[6:7], s[10:11]
 ; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
 ; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
+; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s11, s10
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[6:7], s[10:11]
 ; GCN-IR-NEXT:    s_sub_u32 s8, s6, s10
 ; GCN-IR-NEXT:    s_subb_u32 s9, s7, s10
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[8:9], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
 ; GCN-IR-NEXT:    s_flbit_i32_b64 s12, s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[10:11], s[6:7], s[10:11]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[6:7], s[14:15]
 ; GCN-IR-NEXT:    s_flbit_i32_b64 s20, s[2:3]
 ; GCN-IR-NEXT:    s_sub_u32 s14, s12, s20
 ; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
@@ -1215,17 +1197,17 @@ define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[4:5], 24
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 24
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 16
-; GCN-IR-NEXT:    s_lshl_b64 s[4:5], s[4:5], 16
-; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[2:3], 16
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[4:5], 16
+; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[2:3], 16
 ; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-IR-NEXT:    s_ashr_i32 s10, s5, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[4:5], 16
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_mov_b32 s11, s10
-; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[6:7], s[2:3]
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 16
+; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], s[2:3]
 ; GCN-IR-NEXT:    s_sub_u32 s4, s4, s2
 ; GCN-IR-NEXT:    s_subb_u32 s5, s5, s2
+; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s11, s10
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[10:11]
 ; GCN-IR-NEXT:    s_sub_u32 s6, s6, s10
 ; GCN-IR-NEXT:    s_subb_u32 s7, s7, s10
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir b/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir
index 1652f313347457422ba9a7d1c5d95e64c35c2d9a..0f0dd2374156fb8f7925054355870a18b4cb824b 100644
--- a/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir
+++ b/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck %s
 
 ---
 name:            bar
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll b/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
index 5386ef425dcb58f1d09b2d49897175d4bf492d26..64d4a0cf785013d8ca15729699ebdb164cb42016 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
@@ -17,7 +17,7 @@ define amdgpu_ps float @uniform_phi_with_undef(float inreg %c, float %v, i32 %x,
 ; GCN-NEXT:    s_mov_b32 exec_lo, s2
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %if
-; GCN-NEXT:    s_mov_b32 s2, 2.0
+; GCN-NEXT:    s_mov_b32 s2, 0x40400000
 ; GCN-NEXT:    v_div_scale_f32 v1, s3, s2, s2, v0
 ; GCN-NEXT:    v_rcp_f32_e64 v2, v1
 ; GCN-NEXT:    s_mov_b32 s3, 1.0
@@ -39,7 +39,7 @@ entry:
   br i1 %cc, label %if, label %end
 
 if:
-  %v.if = fdiv float %v, 2.0
+  %v.if = fdiv float %v, 3.0
   br label %end
 
 end:
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
index fc00937e6c8a21e3de0316aa646106756c268b15..721114ece56d1a1bc1f99aa4837ba1cc9117cd82 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
@@ -43,25 +43,6 @@ define i32 @test_tail_call(ptr addrspace(1) %out, ptr addrspace(1) %in) {
   ret i32 %c
 }
 
-declare void @external.varargs(i32, double, i64, ...)
-
-; GCN: error: <unknown>:0:0: in function test_call_varargs void (): unsupported call to variadic function external.varargs
-; R600: in function test_call_varargs{{.*}}: unsupported call to function external.varargs
-define void @test_call_varargs() {
-  call void (i32, double, i64, ...) @external.varargs(i32 42, double 1.0, i64 12, i8 3, i16 1, i32 4, float 1.0, double 2.0)
-  ret void
-}
-
-declare i32 @extern_variadic(...)
-
-; GCN: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported required tail call to function extern_variadic
-; R600: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported call to function extern_variadic
-define i32 @test_tail_call_bitcast_extern_variadic(<4 x float> %arg0, <4 x float> %arg1, i32 %arg2) {
-  %add = fadd <4 x float> %arg0, %arg1
-  %call = tail call i32 @extern_variadic(<4 x float> %add)
-  ret i32 %call
-}
-
 ; R600: in function test_c_call{{.*}}: unsupported call to function defined_function
 define amdgpu_ps i32 @test_c_call_from_shader() {
   %call = call i32 @defined_function(i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
index 769db215cf0349e7509fe85361573e189afb13b5..85c34e036e1fd997197dd51376bfb899e597a5f4 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
@@ -14,12 +14,12 @@ target datalayout = "A5"
 ; GCN-ALLOCA-COUNT-4: buffer_store_dword
 ; GCN-ALLOCA:         buffer_load_dword
 
-; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: s_cmp_eq_u32 s{{[0-9]+}}, 1
 ; GCN-PROMOTE: s_cselect_b64 [[CC1:[^,]+]], -1, 0
+; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
 ; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
-; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
 ; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
@@ -294,10 +294,10 @@ entry:
 
 ; GCN-PROMOTE: s_cmp_eq_u32 s{{[0-9]+}}, 1
 ; GCN-PROMOTE: s_cselect_b64 [[CC1:[^,]+]], -1, 0
-; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
+; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
 ; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
-; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
 ; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index 6fcf5067b0225bca9292fc8cd6c8dc026fd700d3..3bf6c104a02543ea3b8800605e8b06a0bc7fdcbb 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -3395,6 +3395,52 @@ main_body:
   ret void
 }
 
+; Test the interaction between wqm and llvm.amdgcn.init.exec.
+define amdgpu_gs void @wqm_init_exec() {
+; GFX9-W64-LABEL: wqm_init_exec:
+; GFX9-W64:       ; %bb.0: ; %bb
+; GFX9-W64-NEXT:    s_mov_b64 exec, -1
+; GFX9-W64-NEXT:    s_mov_b32 s0, 0
+; GFX9-W64-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-W64-NEXT:    s_mov_b32 s1, s0
+; GFX9-W64-NEXT:    s_mov_b32 s2, s0
+; GFX9-W64-NEXT:    s_mov_b32 s3, s0
+; GFX9-W64-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-W64-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-W64-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-W64-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX9-W64-NEXT:    s_wqm_b64 exec, exec
+; GFX9-W64-NEXT:    ; kill: def $sgpr0 killed $sgpr0 killed $exec
+; GFX9-W64-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-W64-NEXT:    ds_write_b32 v0, v1
+; GFX9-W64-NEXT:    s_endpgm
+;
+; GFX10-W32-LABEL: wqm_init_exec:
+; GFX10-W32:       ; %bb.0: ; %bb
+; GFX10-W32-NEXT:    s_mov_b32 exec_lo, -1
+; GFX10-W32-NEXT:    s_mov_b32 s1, exec_lo
+; GFX10-W32-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT:    s_mov_b32 s0, 0
+; GFX10-W32-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-W32-NEXT:    s_mov_b32 s2, s0
+; GFX10-W32-NEXT:    s_and_b32 exec_lo, exec_lo, s1
+; GFX10-W32-NEXT:    v_mov_b32_e32 v1, v0
+; GFX10-W32-NEXT:    v_mov_b32_e32 v2, v0
+; GFX10-W32-NEXT:    v_mov_b32_e32 v3, v0
+; GFX10-W32-NEXT:    v_mov_b32_e32 v4, s0
+; GFX10-W32-NEXT:    s_mov_b32 s1, s0
+; GFX10-W32-NEXT:    s_mov_b32 s3, s0
+; GFX10-W32-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX10-W32-NEXT:    ds_write_b32 v0, v4
+; GFX10-W32-NEXT:    s_endpgm
+bb:
+  call void @llvm.amdgcn.init.exec(i64 -1)
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> zeroinitializer, <4 x i32> zeroinitializer, i32 0, i32 0, i32 0)
+  %i = call i32 @llvm.amdgcn.wqm.i32(i32 0)
+  store i32 %i, i32 addrspace(3)* null, align 4
+  ret void
+}
+
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
 declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1
 
diff --git a/llvm/test/CodeGen/AMDGPU/xor-r600.ll b/llvm/test/CodeGen/AMDGPU/xor-r600.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3fb11f4484bd26291a5e1f87087af6c5c8136f1a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/xor-r600.ll
@@ -0,0 +1,478 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefixes=R600 %s
+
+define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: xor_v2i32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 3, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_64 T1.XY, T1.X, 0, #1
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV T0.X, KC0[2].Z,
+; R600-NEXT:     MOV * T1.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     XOR_INT * T0.Y, T0.Y, T1.Y,
+; R600-NEXT:     XOR_INT T0.X, T0.X, T1.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load <2 x i32>, ptr addrspace(1) %in0
+  %b = load <2 x i32>, ptr addrspace(1) %in1
+  %result = xor <2 x i32> %a, %b
+  store <2 x i32> %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: xor_v4i32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 5, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_128 T1.XYZW, T1.X, 0, #1
+; R600-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV T0.X, KC0[2].Z,
+; R600-NEXT:     MOV * T1.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     XOR_INT * T0.W, T0.W, T1.W,
+; R600-NEXT:     XOR_INT * T0.Z, T0.Z, T1.Z,
+; R600-NEXT:     XOR_INT * T0.Y, T0.Y, T1.Y,
+; R600-NEXT:     XOR_INT T0.X, T0.X, T1.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load <4 x i32>, ptr addrspace(1) %in0
+  %b = load <4 x i32>, ptr addrspace(1) %in1
+  %result = xor <4 x i32> %a, %b
+  store <4 x i32> %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: xor_i1:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @8
+; R600-NEXT:    ALU 0, @13, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @10
+; R600-NEXT:    ALU 5, @14, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 8:
+; R600-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; R600-NEXT:    Fetch clause starting at 10:
+; R600-NEXT:     VTX_READ_32 T1.X, T1.X, 0, #1
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     MOV * T0.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 13:
+; R600-NEXT:     MOV * T1.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 14:
+; R600-NEXT:     SETGE_DX10 T0.W, T0.X, 1.0,
+; R600-NEXT:     SETGE_DX10 * T1.W, T1.X, 0.0,
+; R600-NEXT:     XOR_INT * T0.W, PS, PV.W,
+; R600-NEXT:     CNDE_INT T0.X, PV.W, T0.X, T1.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load float, ptr addrspace(1) %in0
+  %b = load float, ptr addrspace(1) %in1
+  %acmp = fcmp oge float %a, 0.000000e+00
+  %bcmp = fcmp oge float %b, 1.000000e+00
+  %xor = xor i1 %acmp, %bcmp
+  %result = select i1 %xor, float %a, float %b
+  store float %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: v_xor_i1:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @8
+; R600-NEXT:    ALU 0, @13, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @10
+; R600-NEXT:    ALU 12, @14, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 8:
+; R600-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; R600-NEXT:    Fetch clause starting at 10:
+; R600-NEXT:     VTX_READ_8 T1.X, T1.X, 0, #1
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 13:
+; R600-NEXT:     MOV * T1.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 14:
+; R600-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; R600-NEXT:     XOR_INT * T1.W, T0.X, T1.X,
+; R600-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; R600-NEXT:     AND_INT T1.W, PS, 1,
+; R600-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; R600-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; R600-NEXT:     LSHL T0.X, PV.W, PS,
+; R600-NEXT:     LSHL * T0.W, literal.x, PS,
+; R600-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; R600-NEXT:     MOV T0.Y, 0.0,
+; R600-NEXT:     MOV * T0.Z, 0.0,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load volatile i1, ptr addrspace(1) %in0
+  %b = load volatile i1, ptr addrspace(1) %in1
+  %xor = xor i1 %a, %b
+  store i1 %xor, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: vector_xor_i32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 2, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_32 T1.X, T1.X, 0, #1
+; R600-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV T0.X, KC0[2].Z,
+; R600-NEXT:     MOV * T1.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     XOR_INT T0.X, T0.X, T1.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load i32, ptr addrspace(1) %in0
+  %b = load i32, ptr addrspace(1) %in1
+  %result = xor i32 %a, %b
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
+; R600-LABEL: scalar_xor_i32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:     NOT_INT * T1.X, KC0[2].Z,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %result = xor i32 %a, -1
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: vector_not_i32:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @6
+; R600-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 8:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 9:
+; R600-NEXT:     NOT_INT T0.X, T0.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load i32, ptr addrspace(1) %in0
+  %b = load i32, ptr addrspace(1) %in1
+  %result = xor i32 %a, -1
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: vector_xor_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 3, @12, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_64 T1.XY, T1.X, 0, #1
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV T0.X, KC0[2].Z,
+; R600-NEXT:     MOV * T1.X, KC0[2].W,
+; R600-NEXT:    ALU clause starting at 12:
+; R600-NEXT:     XOR_INT * T0.Y, T0.Y, T1.Y,
+; R600-NEXT:     XOR_INT T0.X, T0.X, T1.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load i64, ptr addrspace(1) %in0
+  %b = load i64, ptr addrspace(1) %in1
+  %result = xor i64 %a, %b
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_i64(ptr addrspace(1) %out, i64 %a, i64 %b) {
+; R600-LABEL: scalar_xor_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     XOR_INT * T0.Y, KC0[3].X, KC0[3].Z,
+; R600-NEXT:     XOR_INT * T0.X, KC0[2].W, KC0[3].Y,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %result = xor i64 %a, %b
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_not_i64(ptr addrspace(1) %out, i64 %a) {
+; R600-LABEL: scalar_not_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     NOT_INT * T0.Y, KC0[3].X,
+; R600-NEXT:     NOT_INT T0.X, KC0[2].W,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %result = xor i64 %a, -1
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; R600-LABEL: vector_not_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @6
+; R600-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 8:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 9:
+; R600-NEXT:     NOT_INT * T0.Y, T0.Y,
+; R600-NEXT:     NOT_INT T0.X, T0.X,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+  %a = load i64, ptr addrspace(1) %in0
+  %b = load i64, ptr addrspace(1) %in1
+  %result = xor i64 %a, -1
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @xor_cf(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %a, i64 %b) {
+; R600-LABEL: xor_cf:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU_PUSH_BEFORE 4, @14, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    JUMP @5 POP:1
+; R600-NEXT:    ALU 0, @19, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @12
+; R600-NEXT:    ALU_POP_AFTER 1, @20, KC0[], KC1[]
+; R600-NEXT:    ALU_PUSH_BEFORE 2, @22, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    JUMP @8 POP:1
+; R600-NEXT:    ALU_POP_AFTER 5, @25, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    ALU 1, @31, KC0[], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 12:
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 14:
+; R600-NEXT:     OR_INT T0.W, KC0[2].W, KC0[3].X,
+; R600-NEXT:     MOV * T1.W, literal.x,
+; R600-NEXT:    1(1.401298e-45), 0(0.000000e+00)
+; R600-NEXT:     SETNE_INT * T0.W, PV.W, 0.0,
+; R600-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PV.W, 0.0,
+; R600-NEXT:    ALU clause starting at 19:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 20:
+; R600-NEXT:     MOV * T1.W, literal.x,
+; R600-NEXT:    0(0.000000e+00), 0(0.000000e+00)
+; R600-NEXT:    ALU clause starting at 22:
+; R600-NEXT:     MOV T0.W, KC0[2].Y,
+; R600-NEXT:     SETE_INT * T1.W, T1.W, 0.0,
+; R600-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
+; R600-NEXT:    ALU clause starting at 25:
+; R600-NEXT:     MOV T1.W, KC0[2].W,
+; R600-NEXT:     MOV * T2.W, KC0[3].Y,
+; R600-NEXT:     XOR_INT T0.X, PV.W, PS,
+; R600-NEXT:     MOV T1.W, KC0[3].X,
+; R600-NEXT:     MOV * T2.W, KC0[3].Z,
+; R600-NEXT:     XOR_INT * T0.Y, PV.W, PS,
+; R600-NEXT:    ALU clause starting at 31:
+; R600-NEXT:     LSHR * T1.X, T0.W, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+entry:
+  %0 = icmp eq i64 %a, 0
+  br i1 %0, label %if, label %else
+
+if:
+  %1 = xor i64 %a, %b
+  br label %endif
+
+else:
+  %2 = load i64, ptr addrspace(1) %in
+  br label %endif
+
+endif:
+  %3 = phi i64 [%1, %if], [%2, %else]
+  store i64 %3, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_literal_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; R600-LABEL: scalar_xor_literal_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     XOR_INT * T0.Y, KC0[5].X, literal.x,
+; R600-NEXT:    992123(1.390260e-39), 0(0.000000e+00)
+; R600-NEXT:     XOR_INT T0.X, KC0[4].W, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    12345(1.729903e-41), 2(2.802597e-45)
+  %or = xor i64 %a, 4261135838621753
+  store i64 %or, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %out, [8 x i32], i64 %a, i64 %b) {
+; R600-LABEL: scalar_xor_literal_multi_use_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 12, @6, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T3.XY, T4.X, 0
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T2.X, 0
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T2.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 6:
+; R600-NEXT:     ADDC_UINT * T0.W, KC0[5].Y, literal.x,
+; R600-NEXT:    12345(1.729903e-41), 0(0.000000e+00)
+; R600-NEXT:     ADD_INT T0.X, KC0[5].Y, literal.x,
+; R600-NEXT:     ADD_INT * T0.W, KC0[5].Z, PV.W,
+; R600-NEXT:    12345(1.729903e-41), 0(0.000000e+00)
+; R600-NEXT:     ADD_INT T1.X, PV.W, literal.x,
+; R600-NEXT:     MOV * T2.X, literal.y,
+; R600-NEXT:    992123(1.390260e-39), 0(0.000000e+00)
+; R600-NEXT:     XOR_INT * T3.Y, KC0[5].X, literal.x,
+; R600-NEXT:    992123(1.390260e-39), 0(0.000000e+00)
+; R600-NEXT:     XOR_INT T3.X, KC0[4].W, literal.x,
+; R600-NEXT:     LSHR * T4.X, KC0[2].Y, literal.y,
+; R600-NEXT:    12345(1.729903e-41), 2(2.802597e-45)
+  %or = xor i64 %a, 4261135838621753
+  store i64 %or, ptr addrspace(1) %out
+
+  %foo = add i64 %b, 4261135838621753
+  store volatile i64 %foo, ptr addrspace(1) undef
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; R600-LABEL: scalar_xor_inline_imm_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     MOV * T0.Y, KC0[5].X,
+; R600-NEXT:     XOR_INT T0.X, KC0[4].W, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    63(8.828180e-44), 2(2.802597e-45)
+  %or = xor i64 %a, 63
+  store i64 %or, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @scalar_xor_neg_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; R600-LABEL: scalar_xor_neg_inline_imm_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     NOT_INT * T0.Y, KC0[5].X,
+; R600-NEXT:     XOR_INT T0.X, KC0[4].W, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    -8(nan), 2(2.802597e-45)
+  %or = xor i64 %a, -8
+  store i64 %or, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_xor_i64_neg_inline_imm(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; R600-LABEL: vector_xor_i64_neg_inline_imm:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @6
+; R600-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 8:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 9:
+; R600-NEXT:     NOT_INT * T0.Y, T0.Y,
+; R600-NEXT:     XOR_INT T0.X, T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    -8(nan), 2(2.802597e-45)
+  %loada = load i64, ptr addrspace(1) %a, align 8
+  %or = xor i64 %loada, -8
+  store i64 %or, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @vector_xor_literal_i64(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; R600-LABEL: vector_xor_literal_i64:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    TEX 0 @6
+; R600-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; R600-NEXT:    ALU clause starting at 8:
+; R600-NEXT:     MOV * T0.X, KC0[2].Z,
+; R600-NEXT:    ALU clause starting at 9:
+; R600-NEXT:     XOR_INT * T0.Y, T0.Y, literal.x,
+; R600-NEXT:    5231(7.330192e-42), 0(0.000000e+00)
+; R600-NEXT:     XOR_INT T0.X, T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    -545810305(-1.784115e+19), 2(2.802597e-45)
+  %loada = load i64, ptr addrspace(1) %a, align 8
+  %or = xor i64 %loada, 22470723082367
+  store i64 %or, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/xor.ll b/llvm/test/CodeGen/AMDGPU/xor.ll
index 1315c0b52af43d9bb7501bf041a3864671d0bd38..e15fd7f29671a4f2afd869945008c515fc3ebdd2 100644
--- a/llvm/test/CodeGen/AMDGPU/xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/xor.ll
@@ -1,16 +1,49 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
-
-
-; FUNC-LABEL: {{^}}xor_v2i32:
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-; SI: v_xor_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_xor_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefixes=SI,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=VI,GCN %s
 
 define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: xor_v2i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v1, v3, v1
+; SI-NEXT:    v_xor_b32_e32 v0, v2, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: xor_v2i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v1, v1, v3
+; VI-NEXT:    v_xor_b32_e32 v0, v0, v2
+; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_endpgm
   %a = load <2 x i32>, ptr addrspace(1) %in0
   %b = load <2 x i32>, ptr addrspace(1) %in1
   %result = xor <2 x i32> %a, %b
@@ -18,18 +51,52 @@ define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in
   ret void
 }
 
-; FUNC-LABEL: {{^}}xor_v4i32:
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
-
 define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: xor_v4i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[12:15], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v3, v7, v3
+; SI-NEXT:    v_xor_b32_e32 v2, v6, v2
+; SI-NEXT:    v_xor_b32_e32 v1, v5, v1
+; SI-NEXT:    v_xor_b32_e32 v0, v4, v0
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: xor_v4i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v8, s4
+; VI-NEXT:    v_mov_b32_e32 v9, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v3, v3, v7
+; VI-NEXT:    v_xor_b32_e32 v2, v2, v6
+; VI-NEXT:    v_xor_b32_e32 v1, v1, v5
+; VI-NEXT:    v_xor_b32_e32 v0, v0, v4
+; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; VI-NEXT:    s_endpgm
   %a = load <4 x i32>, ptr addrspace(1) %in0
   %b = load <4 x i32>, ptr addrspace(1) %in1
   %result = xor <4 x i32> %a, %b
@@ -37,16 +104,54 @@ define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in
   ret void
 }
 
-; FUNC-LABEL: {{^}}xor_i1:
-; EG: XOR_INT {{\** *}}{{T[0-9]+\.[XYZW]}}, {{PS|PV\.[XYZW]}}, {{PS|PV\.[XYZW]}}
-
-; SI-DAG: v_cmp_le_f32_e32 [[CMP0:vcc]], 1.0, {{v[0-9]+}}
-; SI-DAG: v_cmp_le_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]], 0, {{v[0-9]+}}
-; SI: s_xor_b64 [[XOR:vcc]], [[CMP1]], [[CMP0]]
-; SI: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
 define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: xor_i1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s2, s10
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s14, s10
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT:    s_mov_b32 s8, s4
+; SI-NEXT:    s_mov_b32 s9, s5
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cmp_le_f32_e32 vcc, 1.0, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_f32_e64 s[0:1], 0, v1
+; SI-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: xor_i1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_cmp_le_f32_e32 vcc, 0, v4
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_f32_e64 s[0:1], 1.0, v2
+; VI-NEXT:    s_xor_b64 vcc, vcc, s[0:1]
+; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %a = load float, ptr addrspace(1) %in0
   %b = load float, ptr addrspace(1) %in1
   %acmp = fcmp oge float %a, 0.000000e+00
@@ -57,13 +162,50 @@ define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0,
   ret void
 }
 
-; FUNC-LABEL: {{^}}v_xor_i1:
-; SI: buffer_load_ubyte [[B:v[0-9]+]]
-; SI: buffer_load_ubyte [[A:v[0-9]+]]
-; SI: v_xor_b32_e32 [[XOR:v[0-9]+]], [[B]], [[A]]
-; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[XOR]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: v_xor_i1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[12:15], 0 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    v_xor_b32_e32 v0, v0, v1
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_xor_i1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_load_ubyte v4, v[0:1] glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_load_ubyte v2, v[2:3] glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_xor_b32_e32 v2, v4, v2
+; VI-NEXT:    v_and_b32_e32 v2, 1, v2
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
   %xor = xor i1 %a, %b
@@ -71,9 +213,46 @@ define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_xor_i32:
-; SI: v_xor_b32_e32
 define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: vector_xor_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v0, v0, v1
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_xor_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    flat_load_dword v2, v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v2, v4, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %a = load i32, ptr addrspace(1) %in0
   %b = load i32, ptr addrspace(1) %in1
   %result = xor i32 %a, %b
@@ -81,25 +260,96 @@ define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_i32:
-; SI: s_xor_b32
 define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
+; SI-LABEL: scalar_xor_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_xor_b32 s0, s2, s3
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s2, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %result = xor i32 %a, %b
   store i32 %result, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_not_i32:
-; SI: s_not_b32
 define amdgpu_kernel void @scalar_not_i32(ptr addrspace(1) %out, i32 %a) {
+; SI-LABEL: scalar_not_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_not_b32 s4, s4
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_not_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_not_b32 s2, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %result = xor i32 %a, -1
   store i32 %result, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_not_i32:
-; SI: v_not_b32
 define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: vector_not_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_not_b32_e32 v0, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_not_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dword v2, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_not_b32_e32 v2, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %a = load i32, ptr addrspace(1) %in0
   %b = load i32, ptr addrspace(1) %in1
   %result = xor i32 %a, -1
@@ -107,11 +357,48 @@ define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_xor_i64:
-; SI: v_xor_b32_e32
-; SI: v_xor_b32_e32
-; SI: s_endpgm
 define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: vector_xor_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s10, s2
+; SI-NEXT:    s_mov_b32 s11, s3
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s12, s6
+; SI-NEXT:    s_mov_b32 s13, s7
+; SI-NEXT:    s_mov_b32 s14, s2
+; SI-NEXT:    s_mov_b32 s15, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v0, v2, v0
+; SI-NEXT:    v_xor_b32_e32 v1, v3, v1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_xor_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v0, v0, v2
+; VI-NEXT:    v_xor_b32_e32 v1, v1, v3
+; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_endpgm
   %a = load i64, ptr addrspace(1) %in0
   %b = load i64, ptr addrspace(1) %in1
   %result = xor i64 %a, %b
@@ -119,27 +406,104 @@ define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_i64:
-; SI: s_xor_b64
-; SI: s_endpgm
 define amdgpu_kernel void @scalar_xor_i64(ptr addrspace(1) %out, i64 %a, i64 %b) {
+; SI-LABEL: scalar_xor_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_xor_b64 s[4:5], s[6:7], s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    s_xor_b64 s[0:1], s[6:7], s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %result = xor i64 %a, %b
   store i64 %result, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_not_i64:
-; SI: s_not_b64
 define amdgpu_kernel void @scalar_not_i64(ptr addrspace(1) %out, i64 %a) {
+; SI-LABEL: scalar_not_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_not_b64 s[0:1], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_not_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_not_b64 s[0:1], s[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %result = xor i64 %a, -1
   store i64 %result, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_not_i64:
-; SI: v_not_b32
-; SI: v_not_b32
 define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; SI-LABEL: vector_not_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_not_b32_e32 v0, v0
+; SI-NEXT:    v_not_b32_e32 v1, v1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_not_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_not_b32_e32 v0, v0
+; VI-NEXT:    v_not_b32_e32 v1, v1
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %a = load i64, ptr addrspace(1) %in0
   %b = load i64, ptr addrspace(1) %in1
   %result = xor i64 %a, -1
@@ -147,13 +511,65 @@ define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; Test that we have a pattern to match xor inside a branch.
-; Note that in the future the backend may be smart enough to
-; use an SALU instruction for this.
-
-; FUNC-LABEL: {{^}}xor_cf:
-; SI: s_xor_b64
 define amdgpu_kernel void @xor_cf(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %a, i64 %b) {
+; SI-LABEL: xor_cf:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_cmp_ne_u64_e64 s[10:11], s[4:5], 0
+; SI-NEXT:    s_and_b64 vcc, exec, s[10:11]
+; SI-NEXT:    s_cbranch_vccz .LBB12_4
+; SI-NEXT:  ; %bb.1: ; %else
+; SI-NEXT:    s_mov_b32 s15, 0xf000
+; SI-NEXT:    s_mov_b32 s14, -1
+; SI-NEXT:    s_mov_b32 s12, s2
+; SI-NEXT:    s_mov_b32 s13, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[12:15], 0
+; SI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
+; SI-NEXT:    s_cbranch_vccnz .LBB12_3
+; SI-NEXT:  .LBB12_2: ; %if
+; SI-NEXT:    s_xor_b64 s[2:3], s[4:5], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:  .LBB12_3: ; %endif
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB12_4:
+; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT:    s_branch .LBB12_2
+;
+; VI-LABEL: xor_cf:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    s_mov_b64 s[8:9], 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; VI-NEXT:    s_cbranch_scc0 .LBB12_4
+; VI-NEXT:  ; %bb.1: ; %else
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
+; VI-NEXT:    s_cbranch_vccnz .LBB12_3
+; VI-NEXT:  .LBB12_2: ; %if
+; VI-NEXT:    s_xor_b64 s[2:3], s[4:5], s[6:7]
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:  .LBB12_3: ; %endif
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+; VI-NEXT:  .LBB12_4:
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT:    s_branch .LBB12_2
 entry:
   %0 = icmp eq i64 %a, 0
   br i1 %0, label %if, label %else
@@ -172,27 +588,82 @@ endif:
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_literal_i64:
-; SI: s_load_dwordx2 s[[[LO:[0-9]+]]:[[HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
-; SI-DAG: s_xor_b32 s[[RES_HI:[0-9]+]], s{{[0-9]+}}, 0xf237b
-; SI-DAG: s_xor_b32 s[[RES_LO:[0-9]+]], s{{[0-9]+}}, 0x3039
-; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_LO]]
-; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_HI]]
 define amdgpu_kernel void @scalar_xor_literal_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; SI-LABEL: scalar_xor_literal_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s5, s5, 0xf237b
+; SI-NEXT:    s_xor_b32 s4, s4, 0x3039
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_literal_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s3, s3, 0xf237b
+; VI-NEXT:    s_xor_b32 s2, s2, 0x3039
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %or = xor i64 %a, 4261135838621753
   store i64 %or, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_literal_multi_use_i64:
-; SI: s_load_dwordx4 s[[[LO:[0-9]+]]:[[HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}}
-; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0xf237b
-; SI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x3039
-; SI: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s[[[K_LO]]:[[K_HI]]]
-
-; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x3039
-; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0xf237b
 define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %out, [8 x i32], i64 %a, i64 %b) {
+; SI-LABEL: scalar_xor_literal_multi_use_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x13
+; SI-NEXT:    s_movk_i32 s8, 0x3039
+; SI-NEXT:    s_mov_b32 s9, 0xf237b
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    s_add_u32 s0, s2, 0x3039
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_addc_u32 s1, s3, 0xf237b
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_literal_multi_use_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_movk_i32 s2, 0x3039
+; VI-NEXT:    s_mov_b32 s3, 0xf237b
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    s_add_u32 s0, s6, 0x3039
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_addc_u32 s1, s7, 0xf237b
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_endpgm
   %or = xor i64 %a, 4261135838621753
   store i64 %or, ptr addrspace(1) %out
 
@@ -201,51 +672,146 @@ define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %ou
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_inline_imm_i64:
-; SI: s_load_dwordx2 s[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}}
-; SI-NOT: xor_b32
-; SI: s_xor_b32 s[[VAL_LO]], s{{[0-9]+}}, 63
-; SI-NOT: xor_b32
-; SI: v_mov_b32_e32 v[[VLO:[0-9]+]], s{{[0-9]+}}
-; SI-NOT: xor_b32
-; SI: v_mov_b32_e32 v[[VHI:[0-9]+]], s{{[0-9]+}}
-; SI-NOT: xor_b32
-; SI: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]
 define amdgpu_kernel void @scalar_xor_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; SI-LABEL: scalar_xor_inline_imm_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b32 s4, s4, 63
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_inline_imm_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b32 s2, s2, 63
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %or = xor i64 %a, 63
   store i64 %or, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}scalar_xor_neg_inline_imm_i64:
-; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}}
-; SI: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, -8
 define amdgpu_kernel void @scalar_xor_neg_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) {
+; SI-LABEL: scalar_xor_neg_inline_imm_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_xor_b64 s[4:5], s[4:5], -8
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: scalar_xor_neg_inline_imm_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_xor_b64 s[2:3], s[2:3], -8
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %or = xor i64 %a, -8
   store i64 %or, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_xor_i64_neg_inline_imm:
-; SI: buffer_load_dwordx2 v[[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]],
-; SI: v_xor_b32_e32 {{v[0-9]+}}, -8, v[[LO_VREG]]
-; SI: v_xor_b32_e32 {{v[0-9]+}}, -1, {{.*}}
-; SI: s_endpgm
 define amdgpu_kernel void @vector_xor_i64_neg_inline_imm(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; SI-LABEL: vector_xor_i64_neg_inline_imm:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v0, -8, v0
+; SI-NEXT:    v_xor_b32_e32 v1, -1, v1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_xor_i64_neg_inline_imm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v0, -8, v0
+; VI-NEXT:    v_xor_b32_e32 v1, -1, v1
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %loada = load i64, ptr addrspace(1) %a, align 8
   %or = xor i64 %loada, -8
   store i64 %or, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}vector_xor_literal_i64:
-; SI-DAG: buffer_load_dwordx2 v[[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]],
-; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0xdf77987f, v[[LO_VREG]]
-; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0x146f, v[[HI_VREG]]
-; SI: s_endpgm
 define amdgpu_kernel void @vector_xor_literal_i64(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; SI-LABEL: vector_xor_literal_i64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v1, 0x146f, v1
+; SI-NEXT:    v_xor_b32_e32 v0, 0xdf77987f, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: vector_xor_literal_i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v1, 0x146f, v1
+; VI-NEXT:    v_xor_b32_e32 v0, 0xdf77987f, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
   %loada = load i64, ptr addrspace(1) %a, align 8
   %or = xor i64 %loada, 22470723082367
   store i64 %or, ptr addrspace(1) %out
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/ARM/build-attributes.ll b/llvm/test/CodeGen/ARM/build-attributes.ll
index e8c83b92f3f94722b1c9620d4f214d1f214ba2cd..9220f73d94f8bf6b4ac2eabf62680bd224210b08 100644
--- a/llvm/test/CodeGen/ARM/build-attributes.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes.ll
@@ -224,6 +224,10 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON
 
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus -mattr=-vfp2sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON
+
 ; ARMv8-M
 ; RUN: llc < %s -mtriple=thumbv8-none-none-eabi -mcpu=cortex-m23 | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=thumbv8-none-none-eabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
diff --git a/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll b/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll
index c6bff0c9d5ad5096dd7e9f77bbc63f3dc2cbbd51..0cefecd66c71d9b601e6fcddcc52c60628e71d00 100644
--- a/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll
+++ b/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=R52_SCHED
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52plus -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=R52_SCHED
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic    -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
 ;
 ; Check the latency for instructions for both generic and cortex-r52.
diff --git a/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll b/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll
index 6540381d624b8909c4fa94a21e72a80c7f163fb8..a265dd10c0afaa937c380c6e5cdfb61e382131a0 100644
--- a/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll
+++ b/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll
@@ -1,25 +1,24 @@
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all                                2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all      -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf                           2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
+; RUN:     llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none                               2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE
+; RUN:     llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none     -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved                           2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
+; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED
 
 declare void @leaf(i32 %input)
 
 define void @reserved_r7(i32 %input) {
-; RESERVED-NONE-NOT: error: write to reserved register 'R7'
-; RESERVED-R11-NOT: error: write to reserved register 'R7'
+; R7-RESERVED: error: write to reserved register 'R7'
+; R7-FREE-NOT: error: write to reserved register 'R7'
   %1 = call i32 asm sideeffect "mov $0, $1", "={r7},r"(i32 %input)
   ret void
 }
 
 define void @reserved_r11(i32 %input) {
-; RESERVED-NONE-NOT: error: write to reserved register 'R11'
-; RESERVED-R11: error: write to reserved register 'R11'
+; R11-RESERVED: error: write to reserved register 'R11'
+; R11-FREE-NOT: error: write to reserved register 'R11'
   %1 = call i32 asm sideeffect "mov $0, $1", "={r11},r"(i32 %input)
   ret void
 }
diff --git a/llvm/test/CodeGen/ARM/frame-chain.ll b/llvm/test/CodeGen/ARM/frame-chain.ll
index a0f03e51b46137c413b6c6898076cd6928164dee..e37213e4aaf8b8ad9d5797b73c2d29df9454dd70 100644
--- a/llvm/test/CodeGen/ARM/frame-chain.ll
+++ b/llvm/test/CodeGen/ARM/frame-chain.ll
@@ -1,12 +1,11 @@
 ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all | FileCheck %s --check-prefixes=FP,LEAF-FP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS
+; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS
 ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf | FileCheck %s --check-prefixes=FP,LEAF-NOFP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS
+; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS
 ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP
-; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS
+; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS
+; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP
+; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS
 
 define dso_local noundef i32 @leaf(i32 noundef %0) {
 ; LEAF-FP-LABEL: leaf:
diff --git a/llvm/test/CodeGen/ARM/frem-power2.ll b/llvm/test/CodeGen/ARM/frem-power2.ll
index 71c2c09c0105c022a5c86f1f961ee80d7099f50a..63ecd9fec788398935996d1e99809942a590b630 100644
--- a/llvm/test/CodeGen/ARM/frem-power2.ll
+++ b/llvm/test/CodeGen/ARM/frem-power2.ll
@@ -14,26 +14,28 @@ define float @frem4(float %x) {
 ;
 ; CHECK-FP-LABEL: frem4:
 ; CHECK-FP:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    vmov.f32 s0, #4.000000e+00
-; CHECK-FP-NEXT:    vmov s2, r0
+; CHECK-FP-NEXT:    vmov.f32 s0, #2.500000e-01
+; CHECK-FP-NEXT:    vmov.f32 s2, #-4.000000e+00
+; CHECK-FP-NEXT:    vmov s4, r0
 ; CHECK-FP-NEXT:    lsrs r0, r0, #31
-; CHECK-FP-NEXT:    vdiv.f32 s4, s2, s0
-; CHECK-FP-NEXT:    vrintz.f32 s4, s4
-; CHECK-FP-NEXT:    vfms.f32 s2, s4, s0
-; CHECK-FP-NEXT:    vmov r1, s2
+; CHECK-FP-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-FP-NEXT:    vrintz.f32 s0, s0
+; CHECK-FP-NEXT:    vfma.f32 s4, s0, s2
+; CHECK-FP-NEXT:    vmov r1, s4
 ; CHECK-FP-NEXT:    bfi r1, r0, #31, #1
 ; CHECK-FP-NEXT:    mov r0, r1
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-M33-LABEL: frem4:
 ; CHECK-M33:       @ %bb.0: @ %entry
-; CHECK-M33-NEXT:    vmov.f32 s0, #4.000000e+00
-; CHECK-M33-NEXT:    vmov s2, r0
+; CHECK-M33-NEXT:    vmov.f32 s0, #2.500000e-01
+; CHECK-M33-NEXT:    vmov.f32 s2, #-4.000000e+00
+; CHECK-M33-NEXT:    vmov s4, r0
 ; CHECK-M33-NEXT:    lsrs r0, r0, #31
-; CHECK-M33-NEXT:    vdiv.f32 s4, s2, s0
-; CHECK-M33-NEXT:    vrintz.f32 s4, s4
-; CHECK-M33-NEXT:    vmls.f32 s2, s4, s0
-; CHECK-M33-NEXT:    vmov r1, s2
+; CHECK-M33-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-M33-NEXT:    vrintz.f32 s0, s0
+; CHECK-M33-NEXT:    vmla.f32 s4, s0, s2
+; CHECK-M33-NEXT:    vmov r1, s4
 ; CHECK-M33-NEXT:    bfi r1, r0, #31, #1
 ; CHECK-M33-NEXT:    mov r0, r1
 ; CHECK-M33-NEXT:    bx lr
@@ -53,22 +55,24 @@ define float @frem4_nsz(float %x) {
 ;
 ; CHECK-FP-LABEL: frem4_nsz:
 ; CHECK-FP:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    vmov.f32 s0, #4.000000e+00
-; CHECK-FP-NEXT:    vmov s2, r0
-; CHECK-FP-NEXT:    vdiv.f32 s4, s2, s0
-; CHECK-FP-NEXT:    vrintz.f32 s4, s4
-; CHECK-FP-NEXT:    vfms.f32 s2, s4, s0
-; CHECK-FP-NEXT:    vmov r0, s2
+; CHECK-FP-NEXT:    vmov.f32 s0, #2.500000e-01
+; CHECK-FP-NEXT:    vmov.f32 s2, #-4.000000e+00
+; CHECK-FP-NEXT:    vmov s4, r0
+; CHECK-FP-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-FP-NEXT:    vrintz.f32 s0, s0
+; CHECK-FP-NEXT:    vfma.f32 s4, s0, s2
+; CHECK-FP-NEXT:    vmov r0, s4
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-M33-LABEL: frem4_nsz:
 ; CHECK-M33:       @ %bb.0: @ %entry
-; CHECK-M33-NEXT:    vmov.f32 s0, #4.000000e+00
-; CHECK-M33-NEXT:    vmov s2, r0
-; CHECK-M33-NEXT:    vdiv.f32 s4, s2, s0
-; CHECK-M33-NEXT:    vrintz.f32 s4, s4
-; CHECK-M33-NEXT:    vmls.f32 s2, s4, s0
-; CHECK-M33-NEXT:    vmov r0, s2
+; CHECK-M33-NEXT:    vmov.f32 s0, #2.500000e-01
+; CHECK-M33-NEXT:    vmov.f32 s2, #-4.000000e+00
+; CHECK-M33-NEXT:    vmov s4, r0
+; CHECK-M33-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-M33-NEXT:    vrintz.f32 s0, s0
+; CHECK-M33-NEXT:    vmla.f32 s4, s0, s2
+; CHECK-M33-NEXT:    vmov r0, s4
 ; CHECK-M33-NEXT:    bx lr
 entry:
   %fmod = frem nsz float %x, 4.0
diff --git a/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll b/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll
index 0c519f6285dc995f452ad84270258b62308e417a..ea84a3762ec8b1690d18370d005d0c85f6704e01 100644
--- a/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll
+++ b/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll
@@ -4,6 +4,7 @@
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a53 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a57 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF
+; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52plus %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF
 ; Should not generate negated register offset
 
 define void @sintzero(ptr %a) nounwind {
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
index a92c9dd28be5ae7e17b62d0eb6ffb8224a403e84..7634ecd6e863ae370cf0ff918b8e06437e673231 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
@@ -26,15 +26,15 @@ body:             |
   ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r4, -8
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_2
   ; CHECK: bb.1:
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_2
   ; CHECK: bb.2:
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_2
   ; CHECK: bb.3:
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_2
   ; CHECK: bb.4:
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_2
   ; CHECK: bb.5:
   ; CHECK:   $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
   ; CHECK:   BX_RET 14 /* CC::al */, $noreg
@@ -139,13 +139,13 @@ body:             |
   ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r4, -8
-  ; CHECK:   BL @OUTLINED_FUNCTION_1
+  ; CHECK:   BL @OUTLINED_FUNCTION_0
   ; CHECK: bb.1:
-  ; CHECK:   BL @OUTLINED_FUNCTION_1
+  ; CHECK:   BL @OUTLINED_FUNCTION_0
   ; CHECK: bb.2:
-  ; CHECK:   BL @OUTLINED_FUNCTION_1
+  ; CHECK:   BL @OUTLINED_FUNCTION_0
   ; CHECK: bb.3:
-  ; CHECK:   BL @OUTLINED_FUNCTION_1
+  ; CHECK:   BL @OUTLINED_FUNCTION_0
   ; CHECK: bb.4:
   ; CHECK:   $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
   ; CHECK:   BX_RET 14 /* CC::al */, $noreg
@@ -245,19 +245,19 @@ body:             |
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r4, -8
   ; CHECK:   BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
-  ; CHECK:   BL @OUTLINED_FUNCTION_2
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK: bb.1:
   ; CHECK:   BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
-  ; CHECK:   BL @OUTLINED_FUNCTION_2
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK: bb.2:
   ; CHECK:   BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
-  ; CHECK:   BL @OUTLINED_FUNCTION_2
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK: bb.3:
   ; CHECK:   BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
-  ; CHECK:   BL @OUTLINED_FUNCTION_2
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK: bb.4:
   ; CHECK:   BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
-  ; CHECK:   BL @OUTLINED_FUNCTION_2
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK: bb.5:
   ; CHECK:   $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
   ; CHECK:   BX_RET 14 /* CC::al */, $noreg
@@ -307,38 +307,17 @@ body:             |
   bb.0:
     BX_RET 14, $noreg
 
-
   ; CHECK-LABEL: name: OUTLINED_FUNCTION_0
   ; CHECK: bb.0:
-  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
-  ; CHECK:   early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
-  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -8
-  ; CHECK:   BL @bar, implicit-def dead $lr, implicit $sp
-  ; CHECK:   $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
-  ; CHECK:   MOVPCLR 14 /* CC::al */, $noreg
-
-  ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
-  ; CHECK: bb.0:
-  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
-  ; CHECK:   early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
-  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -8
-  ; CHECK:   BL @bar, implicit-def dead $lr, implicit $sp
+  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
   ; CHECK:   $r0 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r1 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r2 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r3 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r4 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
   ; CHECK:   TAILJMPd @bar, implicit $sp
 
-  ; CHECK-LABEL: name: OUTLINED_FUNCTION_2
+  ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
   ; CHECK: bb.0:
   ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
   ; CHECK:   $r0 = MOVi 3, 14 /* CC::al */, $noreg, $noreg
@@ -348,31 +327,28 @@ body:             |
   ; CHECK:   $r4 = MOVi 3, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   MOVPCLR 14 /* CC::al */, $noreg
 
+  ; CHECK-LABEL: name: OUTLINED_FUNCTION_2
+  ; CHECK: bb.0:
+  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+  ; CHECK:   $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   MOVPCLR 14 /* CC::al */, $noreg
+
   ; CHECK-LABEL: name: OUTLINED_FUNCTION_3
   ; CHECK: bb.0:
-  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
-  ; CHECK:   early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
-  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -8
-  ; CHECK:   tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp
+  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
   ; CHECK:   $r0 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r1 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r2 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
   ; CHECK:   tTAILJMPdND @bar, 14 /* CC::al */, $noreg, implicit $sp
 
   ; CHECK-LABEL: name: OUTLINED_FUNCTION_4
   ; CHECK: bb.0:
-  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
-  ; CHECK:   early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
-  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -8
-  ; CHECK:   tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp
+  ; CHECK:   liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
   ; CHECK:   $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
   ; CHECK:   tBX_RET 14 /* CC::al */, $noreg
-
-
-
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-default.mir b/llvm/test/CodeGen/ARM/machine-outliner-default.mir
index 6d0218dbfe636dfe18683e5c00272fecb9227aa7..de2b8f559697651a3b81291806309c2f21c2bfa0 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-default.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-default.mir
@@ -19,17 +19,17 @@ body:             |
   ; CHECK: bb.0:
   ; CHECK:   liveins: $lr
   ; CHECK:   early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK:   $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.1:
   ; CHECK:   liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
   ; CHECK:   early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK:   $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.2:
   ; CHECK:   liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
   ; CHECK:   early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   BL @OUTLINED_FUNCTION_0
+  ; CHECK:   BL @OUTLINED_FUNCTION_1
   ; CHECK:   $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.3:
   ; CHECK:   liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
@@ -73,17 +73,17 @@ body:             |
   ; CHECK: bb.0:
   ; CHECK:   liveins: $lr
   ; CHECK:   early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
   ; CHECK:   $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.1:
   ; CHECK:   liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
   ; CHECK:   early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
   ; CHECK:   $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.2:
   ; CHECK:   liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
   ; CHECK:   early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+  ; CHECK:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
   ; CHECK:   $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
   ; CHECK: bb.3:
   ; CHECK:   liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
@@ -114,6 +114,15 @@ body:             |
 
   ; CHECK-LABEL: name: OUTLINED_FUNCTION_0
   ; CHECK: bb.0:
+  ; CHECK:   liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
+  ; CHECK:   $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   tBX_RET 14 /* CC::al */, $noreg
+
+  ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
+  ; CHECK: bb.0:
   ; CHECK:   liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
   ; CHECK:   $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
@@ -122,15 +131,3 @@ body:             |
   ; CHECK:   $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r5 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   MOVPCLR 14 /* CC::al */, $noreg
-
-  ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
-  ; CHECK: bb.0:
-  ; CHECK:   liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
-  ; CHECK:   $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   tBX_RET 14 /* CC::al */, $noreg
-
-
-
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
index ae5caa5b7c06db17ed390ce7b8438f59c390301a..e71edc8ceb3f608a1d249a8ddae543a7e8910550 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
@@ -18,6 +18,7 @@ body:             |
     liveins: $r0
     ; CHECK-LABEL: name:           CheckAddrMode_i12
     ; CHECK: $r1 = MOVr killed $r0, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
     ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I12:[0-9]+]]
     ; CHECK-NEXT: $r6 = LDRi12 $sp, 4088, 14 /* CC::al */, $noreg
     $r1 = MOVr killed $r0, 14, $noreg, $noreg
@@ -47,6 +48,7 @@ body:             |
     liveins: $r1
     ; CHECK-LABEL: name:           CheckAddrMode3
     ; CHECK: $r0 = MOVr killed $r1, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
     ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I3:[0-9]+]]
     ; CHECK-NEXT: $r6 = LDRSH $sp, $noreg, 248, 14 /* CC::al */, $noreg
     $r0 = MOVr killed $r1, 14, $noreg, $noreg
@@ -76,6 +78,7 @@ body:             |
     liveins: $r2
     ; CHECK-LABEL: name:           CheckAddrMode5
     ; CHECK: $r0 = MOVr killed $r2, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
     ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5:[0-9]+]]
     ; CHECK-NEXT: $d5 = VLDRD $sp, 254, 14 /* CC::al */, $noreg
     $r0 = MOVr killed $r2, 14, $noreg, $noreg
@@ -110,6 +113,7 @@ body:             |
     liveins: $r3
     ; CHECK-LABEL: name:           CheckAddrMode5FP16
     ; CHECK: $r0 = MOVr killed $r3, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
     ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5FP16:[0-9]+]]
     ; CHECK-NEXT: $s6 = VLDRH $sp, 252, 14, $noreg
     $r0 = MOVr killed $r3, 14, $noreg, $noreg
@@ -146,41 +150,29 @@ body:             |
     BX_RET 14, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I5]]
-    ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $d0 = VLDRD $sp, 2, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $d1 = VLDRD $sp, 10, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $d4 = VLDRD $sp, 255, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+    ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+    ;CHECK: $d0 = VLDRD $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $d1 = VLDRD $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $d4 = VLDRD $sp, 253, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I5FP16]]
-    ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $s1 = VLDRH $sp, 4, 14, $noreg
-    ;CHECK-NEXT: $s2 = VLDRH $sp, 12, 14, $noreg
-    ;CHECK-NEXT: $s5 = VLDRH $sp, 244, 14, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+    ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+    ;CHECK: $s1 = VLDRH $sp, 0, 14, $noreg
+    ;CHECK-NEXT: $s2 = VLDRH $sp, 8, 14, $noreg
+    ;CHECK-NEXT: $s5 = VLDRH $sp, 240, 14, $noreg
+    ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I12]]
-    ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $r1 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r2 = LDRi12 $sp, 16, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r5 = LDRi12 $sp, 4094, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+    ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9
+    ;CHECK: $r1 = LDRi12 $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r2 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r5 = LDRi12 $sp, 4086, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I3]]
-    ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $r1 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 16, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 255, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+    ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9
+    ;CHECK: $r1 = LDRSH $sp, $noreg, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 247, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
index 561844484244891dfacf5572dbe3e3ed627cb3e8..f2cdaebdfa8baa153335ced1a9f6ad455304c9a3 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
@@ -19,7 +19,7 @@ body:             |
   bb.0:
     liveins: $r1
     ;CHECK-LABEL: name:           CheckAddrModeT2_i12
-    ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg
+    ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
     ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I12:[0-9]+]]
     ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4088, 14 /* CC::al */, $noreg
     $r0 = tMOVr killed $r1, 14, $noreg
@@ -49,7 +49,7 @@ body:             |
   bb.0:
     liveins: $r1
     ;CHECK-LABEL: name:           CheckAddrModeT2_i8
-    ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+    ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
     ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8:[0-9]+]]
     ;CHECK-NEXT: t2STRHT $r0, $sp, 248, 14 /* CC::al */, $noreg
     $r0 = tMOVr $r1, 14, $noreg
@@ -79,7 +79,7 @@ body:             |
   bb.0:
     liveins: $r1
     ;CHECK-LABEL: name:           CheckAddrModeT2_i8s4
-    ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+    ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
     ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8S4:[0-9]+]]
     ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg
     $r0 = tMOVr $r1, 14, $noreg
@@ -109,7 +109,7 @@ body:             |
   bb.0:
     liveins: $r1
     ;CHECK-LABEL: name:           CheckAddrModeT2_ldrex
-    ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+    ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
     ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[LDREX:[0-9]+]]
     ;CHECK-NEXT: $r1 = t2LDREX $sp, 254, 14 /* CC::al */, $noreg
     $r0 = tMOVr $r1, 14, $noreg
@@ -144,8 +144,10 @@ body:             |
   bb.0:
     liveins: $r0, $r1
     ;CHECK-LABEL: name:           CheckAddrModeT1_s
-    ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[T1_S:[0-9]+]]
+    ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
+    ;CHECK-NEXT: tSTRspi $r0, $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tSTRspi $r0, $sp, 4, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tSTRspi $r0, $sp, 253, 14 /* CC::al */, $noreg
     ;CHECK-NEXT: tSTRspi $r0, $sp, 254, 14 /* CC::al */, $noreg
     $r0 = tMOVr $r1, 14, $noreg
     tBL 14, $noreg, @foo, implicit-def dead $lr, implicit $sp
@@ -181,51 +183,29 @@ body:             |
     BX_RET 14, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[LDREX]]
-    ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $r1 = t2LDREX $sp, 2, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r1 = t2LDREX $sp, 10, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r1 = t2LDREX $sp, 255, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK: $r1 = t2LDREX $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r1 = t2LDREX $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r1 = t2LDREX $sp, 253, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I8]]
-    ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: t2STRHT $r0, $sp, 8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: t2STRHT $r0, $sp, 12, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: t2STRHT $r0, $sp, 255, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK: t2STRHT $r0, $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: t2STRHT $r0, $sp, 4, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: t2STRHT $r0, $sp, 247, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I8S4]]
-    ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
+    ;CHECK: t2STRDi8 $r0, $r1, $sp, 0, 14 /* CC::al */, $noreg
     ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 16, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1012, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $nore
 
     ;CHECK: name:           OUTLINED_FUNCTION_[[I12]]
-    ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 12, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4094, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
-
-    ;CHECK: name:           OUTLINED_FUNCTION_[[T1_S]]
-    ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
-    ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
-    ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
-    ;CHECK-NEXT: tSTRspi $r0, $sp, 2, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: tSTRspi $r0, $sp, 6, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: tSTRspi $r0, $sp, 255, 14 /* CC::al */, $noreg
-    ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+    ;CHECK: $r0 = t2LDRi12 $sp, 0, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4086, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
+
+    ;CHECK: name:           OUTLINED_FUNCTION_[[SHARED]]
+    ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg
+    ;CHECK-NEXT: tTAILJMPdND @foo, 14 /* CC::al */, $noreg, implicit $sp
diff --git a/llvm/test/CodeGen/ARM/misched-fp-basic.ll b/llvm/test/CodeGen/ARM/misched-fp-basic.ll
index 2f672b0cb540febd3471f6ca4de799b81a1fc303..eabede386f1f7c4c636f510a438260e231edd282 100644
--- a/llvm/test/CodeGen/ARM/misched-fp-basic.ll
+++ b/llvm/test/CodeGen/ARM/misched-fp-basic.ll
@@ -5,6 +5,8 @@
 ; RUN:   /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_SWIFT
 ; RUN: llc < %s -mtriple=arm-eabi -mcpu=cortex-r52 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > \
 ; RUN:   /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
+; RUN: llc < %s -mtriple=arm-eabi -mcpu=cortex-r52plus -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > \
+; RUN:   /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
 ;
 ; Check the latency of instructions for processors with sched-models
 ;
diff --git a/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir b/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir
index be5340bd51abbea288148561e11916f9251e5870..0cd20a9f1ff5277fa55e108771c35f599861d1c5 100644
--- a/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir
+++ b/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir
@@ -5,6 +5,8 @@
 # RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_A9
 # RUN: llc -o /dev/null %s -mtriple=thumbv8r-eabi -mcpu=cortex-r52 -run-pass  machine-scheduler  -enable-misched -verify-misched \
 # RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
+# RUN: llc -o /dev/null %s -mtriple=thumbv8r-eabi -mcpu=cortex-r52plus -run-pass  machine-scheduler  -enable-misched -verify-misched \
+# RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
 # REQUIRES: asserts
 --- |
   ; ModuleID = 'foo.ll'
diff --git a/llvm/test/CodeGen/ARM/misched-int-basic.mir b/llvm/test/CodeGen/ARM/misched-int-basic.mir
index 4a52af1942473f5a1e8c848ce917a1bbd3a9ca32..1a0ee9b19bc7251c2d429131e6c55daec05fee29 100644
--- a/llvm/test/CodeGen/ARM/misched-int-basic.mir
+++ b/llvm/test/CodeGen/ARM/misched-int-basic.mir
@@ -4,6 +4,8 @@
 # RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_A9
 # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass  machine-scheduler  -enable-misched -verify-misched \
 # RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
+# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass  machine-scheduler  -enable-misched -verify-misched \
+# RUN:  -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52
 # REQUIRES: asserts
 --- |
   ; ModuleID = 'foo.ll'
diff --git a/llvm/test/CodeGen/ARM/neon_vabd.ll b/llvm/test/CodeGen/ARM/neon_vabd.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8695c3e5f3db9365944ac75fdb4310bdd79294c4
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/neon_vabd.ll
@@ -0,0 +1,618 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s
+
+;
+; SABD
+;
+
+define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: sabd_8b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s8 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <8 x i8> %a to <8 x i16>
+  %b.sext = sext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.sext, %b.sext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  %trunc = trunc <8 x i16> %abs to <8 x i8>
+  ret <8 x i8> %trunc
+}
+
+define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sabd_16b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <16 x i8> %a to <16 x i16>
+  %b.sext = sext <16 x i8> %b to <16 x i16>
+  %sub = sub <16 x i16> %a.sext, %b.sext
+  %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true)
+  %trunc = trunc <16 x i16> %abs to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: sabd_4h:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s16 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <4 x i16> %a to <4 x i32>
+  %b.sext = sext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.sext, %b.sext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  %trunc = trunc <4 x i32> %abs to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: sabd_4h_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vshl.i16 d16, d1, #8
+; CHECK-NEXT:    vshl.i16 d17, d0, #8
+; CHECK-NEXT:    vshr.s16 d16, d16, #8
+; CHECK-NEXT:    vshr.s16 d17, d17, #8
+; CHECK-NEXT:    vabd.s16 d0, d17, d16
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <4 x i8> %a to <4 x i16>
+  %b.sext = sext <4 x i8> %b to <4 x i16>
+  %sub = sub <4 x i16> %a.sext, %b.sext
+  %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true)
+  ret <4 x i16> %abs
+}
+
+define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: sabd_8h:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <8 x i16> %a to <8 x i32>
+  %b.sext = sext <8 x i16> %b to <8 x i32>
+  %sub = sub <8 x i32> %a.sext, %b.sext
+  %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
+  %trunc = trunc <8 x i32> %abs to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: sabd_8h_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.s8 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <8 x i8> %a to <8 x i16>
+  %b.sext = sext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.sext, %b.sext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: sabd_2s:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s32 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <2 x i32> %a to <2 x i64>
+  %b.sext = sext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.sext, %b.sext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  %trunc = trunc <2 x i64> %abs to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: sabd_2s_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vshl.i32 d16, d1, #16
+; CHECK-NEXT:    vshl.i32 d17, d0, #16
+; CHECK-NEXT:    vshr.s32 d16, d16, #16
+; CHECK-NEXT:    vshr.s32 d17, d17, #16
+; CHECK-NEXT:    vabd.s32 d0, d17, d16
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <2 x i16> %a to <2 x i32>
+  %b.sext = sext <2 x i16> %b to <2 x i32>
+  %sub = sub <2 x i32> %a.sext, %b.sext
+  %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true)
+  ret <2 x i32> %abs
+}
+
+define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: sabd_4s:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <4 x i32> %a to <4 x i64>
+  %b.sext = sext <4 x i32> %b to <4 x i64>
+  %sub = sub <4 x i64> %a.sext, %b.sext
+  %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true)
+  %trunc = trunc <4 x i64> %abs to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: sabd_4s_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.s16 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <4 x i16> %a to <4 x i32>
+  %b.sext = sext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.sext, %b.sext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: sabd_2d:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    vmov lr, r1, d1
+; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    vmov r12, r0, d0
+; CHECK-NEXT:    subs lr, lr, r2
+; CHECK-NEXT:    asr r4, r1, #31
+; CHECK-NEXT:    sbcs r1, r1, r3
+; CHECK-NEXT:    sbcs r2, r4, r3, asr #31
+; CHECK-NEXT:    vmov r2, r5, d2
+; CHECK-NEXT:    sbc r3, r4, r3, asr #31
+; CHECK-NEXT:    eor r4, lr, r3, asr #31
+; CHECK-NEXT:    eor r1, r1, r3, asr #31
+; CHECK-NEXT:    subs r4, r4, r3, asr #31
+; CHECK-NEXT:    sbc lr, r1, r3, asr #31
+; CHECK-NEXT:    asr r3, r0, #31
+; CHECK-NEXT:    vmov.32 d1[0], r4
+; CHECK-NEXT:    subs r2, r12, r2
+; CHECK-NEXT:    sbcs r0, r0, r5
+; CHECK-NEXT:    sbcs r1, r3, r5, asr #31
+; CHECK-NEXT:    sbc r1, r3, r5, asr #31
+; CHECK-NEXT:    eor r2, r2, r1, asr #31
+; CHECK-NEXT:    eor r0, r0, r1, asr #31
+; CHECK-NEXT:    subs r2, r2, r1, asr #31
+; CHECK-NEXT:    sbc r0, r0, r1, asr #31
+; CHECK-NEXT:    vmov.32 d0[0], r2
+; CHECK-NEXT:    vmov.32 d1[1], lr
+; CHECK-NEXT:    vmov.32 d0[1], r0
+; CHECK-NEXT:    pop {r4, r5, r11, pc}
+  %a.sext = sext <2 x i64> %a to <2 x i128>
+  %b.sext = sext <2 x i64> %b to <2 x i128>
+  %sub = sub <2 x i128> %a.sext, %b.sext
+  %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
+  %trunc = trunc <2 x i128> %abs to <2 x i64>
+  ret <2 x i64> %trunc
+}
+
+define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: sabd_2d_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.s32 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.sext = sext <2 x i32> %a to <2 x i64>
+  %b.sext = sext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.sext, %b.sext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+;
+; UABD
+;
+
+define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: uabd_8b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u8 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <8 x i8> %a to <8 x i16>
+  %b.zext = zext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.zext, %b.zext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  %trunc = trunc <8 x i16> %abs to <8 x i8>
+  ret <8 x i8> %trunc
+}
+
+define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: uabd_16b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <16 x i8> %a to <16 x i16>
+  %b.zext = zext <16 x i8> %b to <16 x i16>
+  %sub = sub <16 x i16> %a.zext, %b.zext
+  %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true)
+  %trunc = trunc <16 x i16> %abs to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: uabd_4h:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u16 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <4 x i16> %a to <4 x i32>
+  %b.zext = zext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.zext, %b.zext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  %trunc = trunc <4 x i32> %abs to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: uabd_4h_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vbic.i16 d1, #0xff00
+; CHECK-NEXT:    vbic.i16 d0, #0xff00
+; CHECK-NEXT:    vabd.u16 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <4 x i8> %a to <4 x i16>
+  %b.zext = zext <4 x i8> %b to <4 x i16>
+  %sub = sub <4 x i16> %a.zext, %b.zext
+  %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true)
+  ret <4 x i16> %abs
+}
+
+define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: uabd_8h:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <8 x i16> %a to <8 x i32>
+  %b.zext = zext <8 x i16> %b to <8 x i32>
+  %sub = sub <8 x i32> %a.zext, %b.zext
+  %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
+  %trunc = trunc <8 x i32> %abs to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: uabd_8h_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.u8 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <8 x i8> %a to <8 x i16>
+  %b.zext = zext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.zext, %b.zext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: uabd_2s:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u32 d0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <2 x i32> %a to <2 x i64>
+  %b.zext = zext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.zext, %b.zext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  %trunc = trunc <2 x i64> %abs to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: uabd_2s_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0xffff
+; CHECK-NEXT:    vand d17, d1, d16
+; CHECK-NEXT:    vand d16, d0, d16
+; CHECK-NEXT:    vabd.u32 d0, d16, d17
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <2 x i16> %a to <2 x i32>
+  %b.zext = zext <2 x i16> %b to <2 x i32>
+  %sub = sub <2 x i32> %a.zext, %b.zext
+  %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true)
+  ret <2 x i32> %abs
+}
+
+define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: uabd_4s:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <4 x i32> %a to <4 x i64>
+  %b.zext = zext <4 x i32> %b to <4 x i64>
+  %sub = sub <4 x i64> %a.zext, %b.zext
+  %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true)
+  %trunc = trunc <4 x i64> %abs to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: uabd_4s_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.u16 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <4 x i16> %a to <4 x i32>
+  %b.zext = zext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.zext, %b.zext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: uabd_2d:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vmov r0, r12, d3
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    vmov lr, r6, d2
+; CHECK-NEXT:    vmov r4, r5, d0
+; CHECK-NEXT:    subs r0, r2, r0
+; CHECK-NEXT:    sbcs r2, r3, r12
+; CHECK-NEXT:    sbcs r3, r1, #0
+; CHECK-NEXT:    sbc r3, r1, #0
+; CHECK-NEXT:    eor r0, r0, r3, asr #31
+; CHECK-NEXT:    eor r2, r2, r3, asr #31
+; CHECK-NEXT:    subs r0, r0, r3, asr #31
+; CHECK-NEXT:    sbc r2, r2, r3, asr #31
+; CHECK-NEXT:    subs r3, r4, lr
+; CHECK-NEXT:    sbcs r6, r5, r6
+; CHECK-NEXT:    vmov.32 d1[0], r0
+; CHECK-NEXT:    sbcs r5, r1, #0
+; CHECK-NEXT:    sbc r1, r1, #0
+; CHECK-NEXT:    eor r3, r3, r1, asr #31
+; CHECK-NEXT:    subs r0, r3, r1, asr #31
+; CHECK-NEXT:    vmov.32 d0[0], r0
+; CHECK-NEXT:    eor r0, r6, r1, asr #31
+; CHECK-NEXT:    sbc r0, r0, r1, asr #31
+; CHECK-NEXT:    vmov.32 d1[1], r2
+; CHECK-NEXT:    vmov.32 d0[1], r0
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+  %a.zext = zext <2 x i64> %a to <2 x i128>
+  %b.zext = zext <2 x i64> %b to <2 x i128>
+  %sub = sub <2 x i128> %a.zext, %b.zext
+  %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
+  %trunc = trunc <2 x i128> %abs to <2 x i64>
+  ret <2 x i64> %trunc
+}
+
+define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: uabd_2d_promoted_ops:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabdl.u32 q0, d0, d1
+; CHECK-NEXT:    bx lr
+  %a.zext = zext <2 x i32> %a to <2 x i64>
+  %b.zext = zext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.zext, %b.zext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @uabd_v16i8_nuw(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: uabd_v16i8_nuw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vsub.i8 q8, q0, q1
+; CHECK-NEXT:    vabs.s8 q0, q8
+; CHECK-NEXT:    bx lr
+  %sub = sub nuw <16 x i8> %a, %b
+  %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
+  ret <16 x i8> %abs
+}
+
+define <8 x i16> @uabd_v8i16_nuw(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: uabd_v8i16_nuw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vsub.i16 q8, q0, q1
+; CHECK-NEXT:    vabs.s16 q0, q8
+; CHECK-NEXT:    bx lr
+  %sub = sub nuw <8 x i16> %a, %b
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <4 x i32> @uabd_v4i32_nuw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: uabd_v4i32_nuw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vsub.i32 q8, q0, q1
+; CHECK-NEXT:    vabs.s32 q0, q8
+; CHECK-NEXT:    bx lr
+  %sub = sub nuw <4 x i32> %a, %b
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @uabd_v2i64_nuw(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: uabd_v2i64_nuw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vsub.i64 q8, q0, q1
+; CHECK-NEXT:    vshr.s64 q9, q8, #63
+; CHECK-NEXT:    veor q8, q8, q9
+; CHECK-NEXT:    vsub.i64 q0, q8, q9
+; CHECK-NEXT:    bx lr
+  %sub = sub nuw <2 x i64> %a, %b
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sabd_v16i8_nsw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub nsw <16 x i8> %a, %b
+  %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
+  ret <16 x i8> %abs
+}
+
+define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: sabd_v8i16_nsw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub nsw <8 x i16> %a, %b
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: sabd_v4i32_nsw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub nsw <4 x i32> %a, %b
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: sabd_v2i64_nsw:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vsub.i64 q8, q0, q1
+; CHECK-NEXT:    vshr.s64 q9, q8, #63
+; CHECK-NEXT:    veor q8, q8, q9
+; CHECK-NEXT:    vsub.i64 q0, q8, q9
+; CHECK-NEXT:    bx lr
+  %sub = sub nsw <2 x i64> %a, %b
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
+; CHECK-LABEL: smaxmin_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
+
+define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
+; CHECK-LABEL: smaxmin_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %sub = sub <8 x i16> %a, %b
+  ret <8 x i16> %sub
+}
+
+define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
+; CHECK-LABEL: smaxmin_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %sub = sub <4 x i32> %a, %b
+  ret <4 x i32> %sub
+}
+
+define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
+; CHECK-LABEL: smaxmin_v2i64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vmov r1, r0, d3
+; CHECK-NEXT:    mov r12, #0
+; CHECK-NEXT:    vmov r4, lr, d1
+; CHECK-NEXT:    vmov r6, r8, d0
+; CHECK-NEXT:    subs r2, r4, r1
+; CHECK-NEXT:    mov r3, r0
+; CHECK-NEXT:    sbcs r2, lr, r0
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movwlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mov r2, r1
+; CHECK-NEXT:    movne r3, lr
+; CHECK-NEXT:    movne r2, r4
+; CHECK-NEXT:    vmov.32 d17[0], r2
+; CHECK-NEXT:    vmov.32 d17[1], r3
+; CHECK-NEXT:    vmov r2, r3, d2
+; CHECK-NEXT:    subs r5, r2, r6
+; CHECK-NEXT:    sbcs r5, r3, r8
+; CHECK-NEXT:    mov r7, r2
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    movwlt r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    movne r7, r6
+; CHECK-NEXT:    vmov.32 d18[0], r7
+; CHECK-NEXT:    subs r7, r1, r4
+; CHECK-NEXT:    sbcs r7, r0, lr
+; CHECK-NEXT:    mov r7, #0
+; CHECK-NEXT:    movwlt r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    movne r1, r4
+; CHECK-NEXT:    vmov.32 d19[0], r1
+; CHECK-NEXT:    subs r1, r6, r2
+; CHECK-NEXT:    sbcs r1, r8, r3
+; CHECK-NEXT:    movwlt r12, #1
+; CHECK-NEXT:    cmp r12, #0
+; CHECK-NEXT:    movne r2, r6
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    vmov.32 d16[0], r2
+; CHECK-NEXT:    movne r1, r8
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    movne r0, lr
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    vmov.32 d16[1], r1
+; CHECK-NEXT:    movne r3, r8
+; CHECK-NEXT:    vmov.32 d19[1], r0
+; CHECK-NEXT:    vmov.32 d18[1], r3
+; CHECK-NEXT:    vsub.i64 q0, q9, q8
+; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+  %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %sub = sub <2 x i64> %a, %b
+  ret <2 x i64> %sub
+}
+
+define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
+; CHECK-LABEL: umaxmin_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
+
+define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
+; CHECK-LABEL: umaxmin_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %sub = sub <8 x i16> %a, %b
+  ret <8 x i16> %sub
+}
+
+define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
+; CHECK-LABEL: umaxmin_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %sub = sub <4 x i32> %a, %b
+  ret <4 x i32> %sub
+}
+
+define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
+; CHECK-LABEL: umaxmin_v2i64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vqsub.u64 q8, q0, q1
+; CHECK-NEXT:    vqsub.u64 q9, q1, q0
+; CHECK-NEXT:    vsub.i64 q8, q8, q0
+; CHECK-NEXT:    vadd.i64 q9, q0, q9
+; CHECK-NEXT:    vadd.i64 q0, q9, q8
+; CHECK-NEXT:    bx lr
+  %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %sub = sub <2 x i64> %a, %b
+  ret <2 x i64> %sub
+}
+
+define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) {
+; CHECK-LABEL: umaxmin_v16i8_com1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vabd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
diff --git a/llvm/test/CodeGen/ARM/proc-resource-sched.ll b/llvm/test/CodeGen/ARM/proc-resource-sched.ll
index 2dfadda755c6fec79136895d2a96b0ea1ae6b9d7..3bd3bff847eefe41febd83acb151b891187ac775 100644
--- a/llvm/test/CodeGen/ARM/proc-resource-sched.ll
+++ b/llvm/test/CodeGen/ARM/proc-resource-sched.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-R52
+; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52plus -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-R52
 ; REQUIRES: asserts
 
 ; source_filename = "sched-2.c"
diff --git a/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir b/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir
index b7a007f7e3ff0e6fa014887ef7bfdfc6b9a6aeb7..7de911486ddf7c65cc949cb61baf0ca0d4aa42ce 100644
--- a/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir
+++ b/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=armv7-apple-ios -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=armv7-apple-ios -passes=regallocfast -o - %s | FileCheck %s
 
 
 # tBX_RET uses an implicit vreg with a sub-register. That implicit use will
diff --git a/llvm/test/CodeGen/ARM/single-issue-r52.mir b/llvm/test/CodeGen/ARM/single-issue-r52.mir
index 05b167316ca4e6c7779141ac7ba6bfaad1f431e8..084afb6f666c1c271b49efaf489e610e92fbfafa 100644
--- a/llvm/test/CodeGen/ARM/single-issue-r52.mir
+++ b/llvm/test/CodeGen/ARM/single-issue-r52.mir
@@ -1,5 +1,7 @@
 # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass  machine-scheduler  -enable-misched -debug-only=machine-scheduler -misched-topdown 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=TOPDOWN
 # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass  machine-scheduler  -enable-misched -debug-only=machine-scheduler -misched-bottomup 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOTTOMUP
+# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass  machine-scheduler  -enable-misched -debug-only=machine-scheduler -misched-topdown 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=TOPDOWN
+# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass  machine-scheduler  -enable-misched -debug-only=machine-scheduler -misched-bottomup 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOTTOMUP
 # REQUIRES: asserts
 --- |
   ; ModuleID = 'foo.ll'
diff --git a/llvm/test/CodeGen/ARM/useaa.ll b/llvm/test/CodeGen/ARM/useaa.ll
index d70d24b3fd3480368b41d3bdc74a954ece969393..9be0b88b359896e201d6a79399e4e939fd2494d7 100644
--- a/llvm/test/CodeGen/ARM/useaa.ll
+++ b/llvm/test/CodeGen/ARM/useaa.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52plus | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 ; RUN: llc < %s -mtriple=armv7m-eabi -mcpu=cortex-m4 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 ; RUN: llc < %s -mtriple=armv8m-eabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
diff --git a/llvm/test/CodeGen/ARM/vaba.ll b/llvm/test/CodeGen/ARM/vaba.ll
index e4a61ea7d91ffb1f95d6deb7f43ef7088f280595..090eaca96b26f995236f9f87ee7915e5c2474fe5 100644
--- a/llvm/test/CodeGen/ARM/vaba.ll
+++ b/llvm/test/CodeGen/ARM/vaba.ll
@@ -1,8 +1,14 @@
-; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vabas8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabas8:
-;CHECK: vaba.s8
+; CHECK-LABEL: vabas8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.s8 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = load <8 x i8>, ptr %C
@@ -12,8 +18,13 @@ define <8 x i8> @vabas8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i16> @vabas16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabas16:
-;CHECK: vaba.s16
+; CHECK-LABEL: vabas16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.s16 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = load <4 x i16>, ptr %C
@@ -23,8 +34,13 @@ define <4 x i16> @vabas16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <2 x i32> @vabas32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabas32:
-;CHECK: vaba.s32
+; CHECK-LABEL: vabas32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.s32 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = load <2 x i32>, ptr %C
@@ -34,8 +50,13 @@ define <2 x i32> @vabas32(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <8 x i8> @vabau8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabau8:
-;CHECK: vaba.u8
+; CHECK-LABEL: vabau8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.u8 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = load <8 x i8>, ptr %C
@@ -45,8 +66,13 @@ define <8 x i8> @vabau8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i16> @vabau16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabau16:
-;CHECK: vaba.u16
+; CHECK-LABEL: vabau16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.u16 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = load <4 x i16>, ptr %C
@@ -56,8 +82,13 @@ define <4 x i16> @vabau16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <2 x i32> @vabau32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabau32:
-;CHECK: vaba.u32
+; CHECK-LABEL: vabau32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d0, [r0]
+; CHECK-NEXT:    vaba.u32 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = load <2 x i32>, ptr %C
@@ -67,8 +98,13 @@ define <2 x i32> @vabau32(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <16 x i8> @vabaQs8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQs8:
-;CHECK: vaba.s8
+; CHECK-LABEL: vabaQs8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.s8 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <16 x i8>, ptr %A
 	%tmp2 = load <16 x i8>, ptr %B
 	%tmp3 = load <16 x i8>, ptr %C
@@ -78,8 +114,13 @@ define <16 x i8> @vabaQs8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <8 x i16> @vabaQs16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQs16:
-;CHECK: vaba.s16
+; CHECK-LABEL: vabaQs16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.s16 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i16>, ptr %B
 	%tmp3 = load <8 x i16>, ptr %C
@@ -89,8 +130,13 @@ define <8 x i16> @vabaQs16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i32> @vabaQs32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQs32:
-;CHECK: vaba.s32
+; CHECK-LABEL: vabaQs32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.s32 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i32>, ptr %B
 	%tmp3 = load <4 x i32>, ptr %C
@@ -100,8 +146,13 @@ define <4 x i32> @vabaQs32(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <16 x i8> @vabaQu8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQu8:
-;CHECK: vaba.u8
+; CHECK-LABEL: vabaQu8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.u8 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <16 x i8>, ptr %A
 	%tmp2 = load <16 x i8>, ptr %B
 	%tmp3 = load <16 x i8>, ptr %C
@@ -111,8 +162,13 @@ define <16 x i8> @vabaQu8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <8 x i16> @vabaQu16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQu16:
-;CHECK: vaba.u16
+; CHECK-LABEL: vabaQu16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.u16 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i16>, ptr %B
 	%tmp3 = load <8 x i16>, ptr %C
@@ -122,8 +178,13 @@ define <8 x i16> @vabaQu16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i32> @vabaQu32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabaQu32:
-;CHECK: vaba.u32
+; CHECK-LABEL: vabaQu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vaba.u32 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i32>, ptr %B
 	%tmp3 = load <4 x i32>, ptr %C
@@ -149,8 +210,13 @@ declare <8 x i16> @llvm.arm.neon.vabdu.v8i16(<8 x i16>, <8 x i16>) nounwind read
 declare <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i16> @vabals8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabals8:
-;CHECK: vabal.s8
+; CHECK-LABEL: vabals8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.s8 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = load <8 x i8>, ptr %C
@@ -161,8 +227,13 @@ define <8 x i16> @vabals8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i32> @vabals16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabals16:
-;CHECK: vabal.s16
+; CHECK-LABEL: vabals16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.s16 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = load <4 x i16>, ptr %C
@@ -173,8 +244,13 @@ define <4 x i32> @vabals16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <2 x i64> @vabals32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabals32:
-;CHECK: vabal.s32
+; CHECK-LABEL: vabals32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.s32 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i64>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = load <2 x i32>, ptr %C
@@ -185,8 +261,13 @@ define <2 x i64> @vabals32(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <8 x i16> @vabalu8(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabalu8:
-;CHECK: vabal.u8
+; CHECK-LABEL: vabalu8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.u8 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = load <8 x i8>, ptr %C
@@ -197,8 +278,13 @@ define <8 x i16> @vabalu8(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <4 x i32> @vabalu16(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabalu16:
-;CHECK: vabal.u16
+; CHECK-LABEL: vabalu16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.u16 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = load <4 x i16>, ptr %C
@@ -209,8 +295,13 @@ define <4 x i32> @vabalu16(ptr %A, ptr %B, ptr %C) nounwind {
 }
 
 define <2 x i64> @vabalu32(ptr %A, ptr %B, ptr %C) nounwind {
-;CHECK-LABEL: vabalu32:
-;CHECK: vabal.u32
+; CHECK-LABEL: vabalu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r2]
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r0]
+; CHECK-NEXT:    vabal.u32 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i64>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = load <2 x i32>, ptr %C
diff --git a/llvm/test/CodeGen/ARM/vabd.ll b/llvm/test/CodeGen/ARM/vabd.ll
index eb5eed83d4ca3b9a1e833096f9ddd51527d5360c..398f7963535cdf55b1410a20f09618dd9ba5d36d 100644
--- a/llvm/test/CodeGen/ARM/vabd.ll
+++ b/llvm/test/CodeGen/ARM/vabd.ll
@@ -1,8 +1,13 @@
-; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vabds8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabds8:
-;CHECK: vabd.s8
+; CHECK-LABEL: vabds8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.s8 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -10,8 +15,12 @@ define <8 x i8> @vabds8(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i16> @vabds16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabds16:
-;CHECK: vabd.s16
+; CHECK-LABEL: vabds16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.s16 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -19,8 +28,12 @@ define <4 x i16> @vabds16(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x i32> @vabds32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabds32:
-;CHECK: vabd.s32
+; CHECK-LABEL: vabds32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.s32 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -28,8 +41,12 @@ define <2 x i32> @vabds32(ptr %A, ptr %B) nounwind {
 }
 
 define <8 x i8> @vabdu8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdu8:
-;CHECK: vabd.u8
+; CHECK-LABEL: vabdu8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.u8 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -37,8 +54,12 @@ define <8 x i8> @vabdu8(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i16> @vabdu16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdu16:
-;CHECK: vabd.u16
+; CHECK-LABEL: vabdu16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.u16 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -46,8 +67,12 @@ define <4 x i16> @vabdu16(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x i32> @vabdu32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdu32:
-;CHECK: vabd.u32
+; CHECK-LABEL: vabdu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.u32 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -55,8 +80,12 @@ define <2 x i32> @vabdu32(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x float> @vabdf32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdf32:
-;CHECK: vabd.f32
+; CHECK-LABEL: vabdf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabd.f32 d0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x float>, ptr %A
 	%tmp2 = load <2 x float>, ptr %B
 	%tmp3 = call <2 x float> @llvm.arm.neon.vabds.v2f32(<2 x float> %tmp1, <2 x float> %tmp2)
@@ -64,8 +93,12 @@ define <2 x float> @vabdf32(ptr %A, ptr %B) nounwind {
 }
 
 define <16 x i8> @vabdQs8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQs8:
-;CHECK: vabd.s8
+; CHECK-LABEL: vabdQs8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.s8 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <16 x i8>, ptr %A
 	%tmp2 = load <16 x i8>, ptr %B
 	%tmp3 = call <16 x i8> @llvm.arm.neon.vabds.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -73,8 +106,12 @@ define <16 x i8> @vabdQs8(ptr %A, ptr %B) nounwind {
 }
 
 define <8 x i16> @vabdQs16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQs16:
-;CHECK: vabd.s16
+; CHECK-LABEL: vabdQs16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.s16 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i16>, ptr %B
 	%tmp3 = call <8 x i16> @llvm.arm.neon.vabds.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -82,8 +119,12 @@ define <8 x i16> @vabdQs16(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i32> @vabdQs32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQs32:
-;CHECK: vabd.s32
+; CHECK-LABEL: vabdQs32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.s32 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i32>, ptr %B
 	%tmp3 = call <4 x i32> @llvm.arm.neon.vabds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -91,8 +132,12 @@ define <4 x i32> @vabdQs32(ptr %A, ptr %B) nounwind {
 }
 
 define <16 x i8> @vabdQu8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQu8:
-;CHECK: vabd.u8
+; CHECK-LABEL: vabdQu8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.u8 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <16 x i8>, ptr %A
 	%tmp2 = load <16 x i8>, ptr %B
 	%tmp3 = call <16 x i8> @llvm.arm.neon.vabdu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -100,8 +145,12 @@ define <16 x i8> @vabdQu8(ptr %A, ptr %B) nounwind {
 }
 
 define <8 x i16> @vabdQu16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQu16:
-;CHECK: vabd.u16
+; CHECK-LABEL: vabdQu16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.u16 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i16>, ptr %A
 	%tmp2 = load <8 x i16>, ptr %B
 	%tmp3 = call <8 x i16> @llvm.arm.neon.vabdu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -109,8 +158,12 @@ define <8 x i16> @vabdQu16(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i32> @vabdQu32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQu32:
-;CHECK: vabd.u32
+; CHECK-LABEL: vabdQu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.u32 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i32>, ptr %A
 	%tmp2 = load <4 x i32>, ptr %B
 	%tmp3 = call <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -118,8 +171,12 @@ define <4 x i32> @vabdQu32(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x float> @vabdQf32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdQf32:
-;CHECK: vabd.f32
+; CHECK-LABEL: vabdQf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vabd.f32 q0, q9, q8
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x float>, ptr %A
 	%tmp2 = load <4 x float>, ptr %B
 	%tmp3 = call <4 x float> @llvm.arm.neon.vabds.v4f32(<4 x float> %tmp1, <4 x float> %tmp2)
@@ -147,8 +204,12 @@ declare <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32>, <4 x i32>) nounwind read
 declare <4 x float> @llvm.arm.neon.vabds.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
 define <8 x i16> @vabdls8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdls8:
-;CHECK: vabdl.s8
+; CHECK-LABEL: vabdls8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.s8 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -157,8 +218,12 @@ define <8 x i16> @vabdls8(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i32> @vabdls16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdls16:
-;CHECK: vabdl.s16
+; CHECK-LABEL: vabdls16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.s16 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -167,8 +232,12 @@ define <4 x i32> @vabdls16(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x i64> @vabdls32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdls32:
-;CHECK: vabdl.s32
+; CHECK-LABEL: vabdls32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.s32 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -177,8 +246,12 @@ define <2 x i64> @vabdls32(ptr %A, ptr %B) nounwind {
 }
 
 define <8 x i16> @vabdlu8(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdlu8:
-;CHECK: vabdl.u8
+; CHECK-LABEL: vabdlu8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.u8 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <8 x i8>, ptr %A
 	%tmp2 = load <8 x i8>, ptr %B
 	%tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -187,8 +260,12 @@ define <8 x i16> @vabdlu8(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i32> @vabdlu16(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdlu16:
-;CHECK: vabdl.u16
+; CHECK-LABEL: vabdlu16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.u16 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
 	%tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -197,8 +274,12 @@ define <4 x i32> @vabdlu16(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x i64> @vabdlu32(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: vabdlu32:
-;CHECK: vabdl.u32
+; CHECK-LABEL: vabdlu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vabdl.u32 q0, d17, d16
+; CHECK-NEXT:    bx lr
 	%tmp1 = load <2 x i32>, ptr %A
 	%tmp2 = load <2 x i32>, ptr %B
 	%tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
diff --git a/llvm/test/CodeGen/ARM/vdiv_combine.ll b/llvm/test/CodeGen/ARM/vdiv_combine.ll
index 988844661085edb76d918a8c3ae10bc7754a7448..899487f9efb2c88617d334bea9382ffe234be002 100644
--- a/llvm/test/CodeGen/ARM/vdiv_combine.ll
+++ b/llvm/test/CodeGen/ARM/vdiv_combine.ll
@@ -5,10 +5,7 @@
 define arm_aapcs_vfpcc <2 x float> @t1(<2 x i32> %vecinit2.i) nounwind {
 ; CHECK-LABEL: t1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.f32 s2, #8.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 d2, d0
-; CHECK-NEXT:    vdiv.f32 s1, s5, s2
-; CHECK-NEXT:    vdiv.f32 s0, s4, s2
+; CHECK-NEXT:    vcvt.f32.s32 d0, d0, #3
 ; CHECK-NEXT:    bx lr
 entry:
   %vcvt.i = sitofp <2 x i32> %vecinit2.i to <2 x float>
@@ -20,10 +17,7 @@ entry:
 define arm_aapcs_vfpcc <2 x float> @t2(<2 x i32> %vecinit2.i) nounwind {
 ; CHECK-LABEL: t2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.f32 s2, #8.000000e+00
-; CHECK-NEXT:    vcvt.f32.u32 d2, d0
-; CHECK-NEXT:    vdiv.f32 s1, s5, s2
-; CHECK-NEXT:    vdiv.f32 s0, s4, s2
+; CHECK-NEXT:    vcvt.f32.u32 d0, d0, #3
 ; CHECK-NEXT:    bx lr
 entry:
   %vcvt.i = uitofp <2 x i32> %vecinit2.i to <2 x float>
@@ -56,17 +50,10 @@ entry:
 define arm_aapcs_vfpcc <2 x float> @t4(<2 x i32> %vecinit2.i) nounwind {
 ; CHECK-LABEL: t4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvt.f32.s32 d2, d0
-; CHECK-NEXT:    vldr s2, LCPI3_0
-; CHECK-NEXT:    vdiv.f32 s1, s5, s2
-; CHECK-NEXT:    vdiv.f32 s0, s4, s2
+; CHECK-NEXT:    vcvt.f32.s32 d16, d0
+; CHECK-NEXT:    vmov.i32 d17, #0x2f000000
+; CHECK-NEXT:    vmul.f32 d0, d16, d17
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:    .data_region
-; CHECK-NEXT:  LCPI3_0:
-; CHECK-NEXT:    .long 0x50000000 @ float 8.58993459E+9
-; CHECK-NEXT:    .end_data_region
 entry:
   %vcvt.i = sitofp <2 x i32> %vecinit2.i to <2 x float>
   %div.i = fdiv <2 x float> %vcvt.i, <float 0x4200000000000000, float 0x4200000000000000>
@@ -77,17 +64,8 @@ entry:
 define arm_aapcs_vfpcc <2 x float> @t5(<2 x i32> %vecinit2.i) nounwind {
 ; CHECK-LABEL: t5:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvt.f32.s32 d2, d0
-; CHECK-NEXT:    vldr s2, LCPI4_0
-; CHECK-NEXT:    vdiv.f32 s1, s5, s2
-; CHECK-NEXT:    vdiv.f32 s0, s4, s2
+; CHECK-NEXT:    vcvt.f32.s32 d0, d0, #32
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:    .data_region
-; CHECK-NEXT:  LCPI4_0:
-; CHECK-NEXT:    .long 0x4f800000 @ float 4.2949673E+9
-; CHECK-NEXT:    .end_data_region
 entry:
   %vcvt.i = sitofp <2 x i32> %vecinit2.i to <2 x float>
   %div.i = fdiv <2 x float> %vcvt.i, <float 0x41F0000000000000, float 0x41F0000000000000>
@@ -98,12 +76,7 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @t6(<4 x i32> %vecinit6.i) nounwind {
 ; CHECK-LABEL: t6:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.f32 s4, #8.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 q2, q0
-; CHECK-NEXT:    vdiv.f32 s3, s11, s4
-; CHECK-NEXT:    vdiv.f32 s2, s10, s4
-; CHECK-NEXT:    vdiv.f32 s1, s9, s4
-; CHECK-NEXT:    vdiv.f32 s0, s8, s4
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0, #3
 ; CHECK-NEXT:    bx lr
 entry:
   %vcvt.i = sitofp <4 x i32> %vecinit6.i to <4 x float>
@@ -115,12 +88,7 @@ define arm_aapcs_vfpcc <4 x float> @fix_unsigned_i16_to_float(<4 x i16> %in) {
 ; CHECK-LABEL: fix_unsigned_i16_to_float:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmovl.u16 q8, d0
-; CHECK-NEXT:    vmov.f32 s4, #2.000000e+00
-; CHECK-NEXT:    vcvt.f32.u32 q2, q8
-; CHECK-NEXT:    vdiv.f32 s3, s11, s4
-; CHECK-NEXT:    vdiv.f32 s2, s10, s4
-; CHECK-NEXT:    vdiv.f32 s1, s9, s4
-; CHECK-NEXT:    vdiv.f32 s0, s8, s4
+; CHECK-NEXT:    vcvt.f32.u32 q0, q8, #1
 ; CHECK-NEXT:    bx lr
   %conv = uitofp <4 x i16> %in to <4 x float>
   %shift = fdiv <4 x float> %conv, <float 2.0, float 2.0, float 2.0, float 2.0>
@@ -131,12 +99,7 @@ define arm_aapcs_vfpcc <4 x float> @fix_signed_i16_to_float(<4 x i16> %in) {
 ; CHECK-LABEL: fix_signed_i16_to_float:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmovl.s16 q8, d0
-; CHECK-NEXT:    vmov.f32 s4, #2.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 q2, q8
-; CHECK-NEXT:    vdiv.f32 s3, s11, s4
-; CHECK-NEXT:    vdiv.f32 s2, s10, s4
-; CHECK-NEXT:    vdiv.f32 s1, s9, s4
-; CHECK-NEXT:    vdiv.f32 s0, s8, s4
+; CHECK-NEXT:    vcvt.f32.s32 q0, q8, #1
 ; CHECK-NEXT:    bx lr
   %conv = sitofp <4 x i16> %in to <4 x float>
   %shift = fdiv <4 x float> %conv, <float 2.0, float 2.0, float 2.0, float 2.0>
@@ -152,13 +115,12 @@ define arm_aapcs_vfpcc <2 x float> @fix_i64_to_float(<2 x i64> %in) {
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl ___floatundisf
 ; CHECK-NEXT:    vmov r2, r1, d8
-; CHECK-NEXT:    vmov s18, r0
-; CHECK-NEXT:    vmov.f32 s16, #2.000000e+00
+; CHECK-NEXT:    vmov s19, r0
+; CHECK-NEXT:    vmov.i32 d8, #0x3f000000
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl ___floatundisf
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    vdiv.f32 s1, s18, s16
-; CHECK-NEXT:    vdiv.f32 s0, s2, s16
+; CHECK-NEXT:    vmov s18, r0
+; CHECK-NEXT:    vmul.f32 d0, d9, d8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {lr}
 ; CHECK-NEXT:    bx lr
@@ -177,13 +139,13 @@ define arm_aapcs_vfpcc <2 x double> @fix_i64_to_double(<2 x i64> %in) {
 ; CHECK-NEXT:    bl ___floatundidf
 ; CHECK-NEXT:    vmov r2, r3, d8
 ; CHECK-NEXT:    vmov d9, r0, r1
-; CHECK-NEXT:    vmov.f64 d8, #2.000000e+00
+; CHECK-NEXT:    vmov.f64 d8, #5.000000e-01
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    bl ___floatundidf
 ; CHECK-NEXT:    vmov d16, r0, r1
-; CHECK-NEXT:    vdiv.f64 d1, d9, d8
-; CHECK-NEXT:    vdiv.f64 d0, d16, d8
+; CHECK-NEXT:    vmul.f64 d1, d9, d8
+; CHECK-NEXT:    vmul.f64 d0, d16, d8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {lr}
 ; CHECK-NEXT:    bx lr
@@ -196,19 +158,8 @@ define arm_aapcs_vfpcc <2 x double> @fix_i64_to_double(<2 x i64> %in) {
 define arm_aapcs_vfpcc <8 x float> @test7(<8 x i32> %in) nounwind {
 ; CHECK-LABEL: test7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.f32 s12, #8.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 q4, q0
-; CHECK-NEXT:    vcvt.f32.s32 q2, q1
-; CHECK-NEXT:    vdiv.f32 s3, s19, s12
-; CHECK-NEXT:    vdiv.f32 s7, s11, s12
-; CHECK-NEXT:    vdiv.f32 s2, s18, s12
-; CHECK-NEXT:    vdiv.f32 s6, s10, s12
-; CHECK-NEXT:    vdiv.f32 s1, s17, s12
-; CHECK-NEXT:    vdiv.f32 s5, s9, s12
-; CHECK-NEXT:    vdiv.f32 s0, s16, s12
-; CHECK-NEXT:    vdiv.f32 s4, s8, s12
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0, #3
+; CHECK-NEXT:    vcvt.f32.s32 q1, q1, #3
 ; CHECK-NEXT:    bx lr
 entry:
   %vcvt.i = sitofp <8 x i32> %in to <8 x float>
@@ -220,19 +171,8 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @test8(<4 x i32> %in) {
 ; CHECK-LABEL: test8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.f32 s4, #2.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 q2, q0
-; CHECK-NEXT:    vdiv.f32 s2, s10, s4
-; CHECK-NEXT:    vdiv.f32 s1, s9, s4
-; CHECK-NEXT:    vdiv.f32 s0, s8, s4
-; CHECK-NEXT:    vldr s3, LCPI11_0
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0, #1
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:    .data_region
-; CHECK-NEXT:  LCPI11_0:
-; CHECK-NEXT:    .long 0x7fc00000 @ float NaN
-; CHECK-NEXT:    .end_data_region
   %vcvt.i = sitofp <4 x i32> %in to <4 x float>
   %div.i = fdiv <4 x float> %vcvt.i, <float 2.0, float 2.0, float 2.0, float undef>
   ret <4 x float> %div.i
@@ -241,19 +181,8 @@ define arm_aapcs_vfpcc <4 x float> @test8(<4 x i32> %in) {
 define arm_aapcs_vfpcc <3 x float> @test_illegal_int_to_fp(<3 x i32> %in) {
 ; CHECK-LABEL: test_illegal_int_to_fp:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.f32 s4, #4.000000e+00
-; CHECK-NEXT:    vcvt.f32.s32 q2, q0
-; CHECK-NEXT:    vdiv.f32 s2, s10, s4
-; CHECK-NEXT:    vdiv.f32 s1, s9, s4
-; CHECK-NEXT:    vdiv.f32 s0, s8, s4
-; CHECK-NEXT:    vldr s3, LCPI12_0
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0, #2
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:    .data_region
-; CHECK-NEXT:  LCPI12_0:
-; CHECK-NEXT:    .long 0x7fc00000 @ float NaN
-; CHECK-NEXT:    .end_data_region
   %conv = sitofp <3 x i32> %in to <3 x float>
   %res = fdiv <3 x float> %conv, <float 4.0, float 4.0, float 4.0>
   ret <3 x float> %res
diff --git a/llvm/test/CodeGen/ARM/vector-store.ll b/llvm/test/CodeGen/ARM/vector-store.ll
index a8a1031637afc4d10544000ab6efd99f8f01662f..a0a801d2b6d2cf520b994991a33d797e61cf517d 100644
--- a/llvm/test/CodeGen/ARM/vector-store.ll
+++ b/llvm/test/CodeGen/ARM/vector-store.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
-
-target datalayout = "e-m:o-p:32:32-i1:8:32-i8:8:32-i16:16:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
-target triple = "thumbv7-none-eabi"
+; RUN: llc < %s -mtriple=thumbv7-none-eabi | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc < %s -mtriple=thumbebv7-none-eabi | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
 define void @store_v8i8(ptr %ptr, <8 x i8> %val) {
 ; CHECK-LABEL: store_v8i8:
@@ -11,24 +9,33 @@ define void @store_v8i8(ptr %ptr, <8 x i8> %val) {
 ; CHECK-NEXT:    str r3, [r0, #4]
 ; CHECK-NEXT:    str r2, [r0]
 ; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <8 x i8> %val, ptr %A, align 1
-	ret void
+    %A = load ptr, ptr %ptr
+    store  <8 x i8> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v8i8_update(ptr %ptr, <8 x i8> %val) {
-; CHECK-LABEL: store_v8i8_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <8 x i8> %val, ptr %A, align 1
-	%inc = getelementptr <8 x i8>, ptr %A, i38 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v8i8_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v8i8_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 d16, d16
+; CHECK-BE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <8 x i8> %val, ptr %A, align 1
+    %inc = getelementptr <8 x i8>, ptr %A, i38 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v4i16(ptr %ptr, <4 x i16> %val) {
@@ -38,24 +45,33 @@ define void @store_v4i16(ptr %ptr, <4 x i16> %val) {
 ; CHECK-NEXT:    str r3, [r0, #4]
 ; CHECK-NEXT:    str r2, [r0]
 ; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x i16> %val, ptr %A, align 1
-	ret void
+    %A = load ptr, ptr %ptr
+    store  <4 x i16> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v4i16_update(ptr %ptr, <4 x i16> %val) {
-; CHECK-LABEL: store_v4i16_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x i16> %val, ptr %A, align 1
-	%inc = getelementptr <4 x i16>, ptr %A, i34 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v4i16_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4i16_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 d16, d16
+; CHECK-BE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <4 x i16> %val, ptr %A, align 1
+    %inc = getelementptr <4 x i16>, ptr %A, i34 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i32(ptr %ptr, <2 x i32> %val) {
@@ -65,24 +81,33 @@ define void @store_v2i32(ptr %ptr, <2 x i32> %val) {
 ; CHECK-NEXT:    str r3, [r0, #4]
 ; CHECK-NEXT:    str r2, [r0]
 ; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i32> %val, ptr %A, align 1
-	ret void
+    %A = load ptr, ptr %ptr
+    store  <2 x i32> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v2i32_update(ptr %ptr, <2 x i32> %val) {
-; CHECK-LABEL: store_v2i32_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i32> %val, ptr %A, align 1
-	%inc = getelementptr <2 x i32>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i32_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i32_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 d16, d16
+; CHECK-BE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i32> %val, ptr %A, align 1
+    %inc = getelementptr <2 x i32>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2f32(ptr %ptr, <2 x float> %val) {
@@ -92,24 +117,33 @@ define void @store_v2f32(ptr %ptr, <2 x float> %val) {
 ; CHECK-NEXT:    str r3, [r0, #4]
 ; CHECK-NEXT:    str r2, [r0]
 ; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x float> %val, ptr %A, align 1
-	ret void
+    %A = load ptr, ptr %ptr
+    store  <2 x float> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v2f32_update(ptr %ptr, <2 x float> %val) {
-; CHECK-LABEL: store_v2f32_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x float> %val, ptr %A, align 1
-	%inc = getelementptr <2 x float>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2f32_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2f32_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 d16, d16
+; CHECK-BE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x float> %val, ptr %A, align 1
+    %inc = getelementptr <2 x float>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v1i64(ptr %ptr, <1 x i64> %val) {
@@ -119,279 +153,458 @@ define void @store_v1i64(ptr %ptr, <1 x i64> %val) {
 ; CHECK-NEXT:    str r3, [r0, #4]
 ; CHECK-NEXT:    str r2, [r0]
 ; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <1 x i64> %val, ptr %A, align 1
-	ret void
+    %A = load ptr, ptr %ptr
+    store  <1 x i64> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v1i64_update(ptr %ptr, <1 x i64> %val) {
-; CHECK-LABEL: store_v1i64_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <1 x i64> %val, ptr %A, align 1
-	%inc = getelementptr <1 x i64>, ptr %A, i31 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v1i64_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v1i64_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 d16, d16
+; CHECK-BE-NEXT:    vst1.8 {d16}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <1 x i64> %val, ptr %A, align 1
+    %inc = getelementptr <1 x i64>, ptr %A, i31 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v16i8(ptr %ptr, <16 x i8> %val) {
-; CHECK-LABEL: store_v16i8:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <16 x i8> %val, ptr %A, align 1
-	ret void
+; CHECK-LE-LABEL: store_v16i8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v16i8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <16 x i8> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v16i8_update(ptr %ptr, <16 x i8> %val) {
-; CHECK-LABEL: store_v16i8_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <16 x i8> %val, ptr %A, align 1
-	%inc = getelementptr <16 x i8>, ptr %A, i316 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v16i8_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v16i8_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <16 x i8> %val, ptr %A, align 1
+    %inc = getelementptr <16 x i8>, ptr %A, i316 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v8i16(ptr %ptr, <8 x i16> %val) {
-; CHECK-LABEL: store_v8i16:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <8 x i16> %val, ptr %A, align 1
-	ret void
+; CHECK-LE-LABEL: store_v8i16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v8i16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <8 x i16> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v8i16_update(ptr %ptr, <8 x i16> %val) {
-; CHECK-LABEL: store_v8i16_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <8 x i16> %val, ptr %A, align 1
-	%inc = getelementptr <8 x i16>, ptr %A, i38 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v8i16_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v8i16_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <8 x i16> %val, ptr %A, align 1
+    %inc = getelementptr <8 x i16>, ptr %A, i38 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v4i32(ptr %ptr, <4 x i32> %val) {
-; CHECK-LABEL: store_v4i32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x i32> %val, ptr %A, align 1
-	ret void
+; CHECK-LE-LABEL: store_v4i32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4i32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <4 x i32> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v4i32_update(ptr %ptr, <4 x i32> %val) {
-; CHECK-LABEL: store_v4i32_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x i32> %val, ptr %A, align 1
-	%inc = getelementptr <4 x i32>, ptr %A, i34 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v4i32_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4i32_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <4 x i32> %val, ptr %A, align 1
+    %inc = getelementptr <4 x i32>, ptr %A, i34 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v4f32(ptr %ptr, <4 x float> %val) {
-; CHECK-LABEL: store_v4f32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x float> %val, ptr %A, align 1
-	ret void
+; CHECK-LE-LABEL: store_v4f32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4f32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <4 x float> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v4f32_update(ptr %ptr, <4 x float> %val) {
-; CHECK-LABEL: store_v4f32_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <4 x float> %val, ptr %A, align 1
-	%inc = getelementptr <4 x float>, ptr %A, i34 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v4f32_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v4f32_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <4 x float> %val, ptr %A, align 1
+    %inc = getelementptr <4 x float>, ptr %A, i34 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i64(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 1
-	ret void
+; CHECK-LE-LABEL: store_v2i64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 1
+    ret void
 }
 
 define void @store_v2i64_update(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.8 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 1
-	%inc = getelementptr <2 x i64>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i64_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.8 q8, q8
+; CHECK-BE-NEXT:    vst1.8 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 1
+    %inc = getelementptr <2 x i64>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i64_update_aligned2(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64_update_aligned2:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.16 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 2
-	%inc = getelementptr <2 x i64>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i64_update_aligned2:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.16 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64_update_aligned2:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q8, q8
+; CHECK-BE-NEXT:    vst1.16 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 2
+    %inc = getelementptr <2 x i64>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i64_update_aligned4(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64_update_aligned4:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.32 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 4
-	%inc = getelementptr <2 x i64>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i64_update_aligned4:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.32 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64_update_aligned4:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vst1.32 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 4
+    %inc = getelementptr <2 x i64>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i64_update_aligned8(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64_update_aligned8:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.64 {d16, d17}, [r1]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 8
-	%inc = getelementptr <2 x i64>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i64_update_aligned8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.64 {d16, d17}, [r1]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64_update_aligned8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vst1.64 {d16, d17}, [r1]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 8
+    %inc = getelementptr <2 x i64>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @store_v2i64_update_aligned16(ptr %ptr, <2 x i64> %val) {
-; CHECK-LABEL: store_v2i64_update_aligned16:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    vst1.64 {d16, d17}, [r1:128]!
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-	store  <2 x i64> %val, ptr %A, align 16
-	%inc = getelementptr <2 x i64>, ptr %A, i32 1
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: store_v2i64_update_aligned16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    vst1.64 {d16, d17}, [r1:128]!
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: store_v2i64_update_aligned16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vst1.64 {d16, d17}, [r1:128]!
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    store  <2 x i64> %val, ptr %A, align 16
+    %inc = getelementptr <2 x i64>, ptr %A, i32 1
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define void @truncstore_v4i32tov4i8(ptr %ptr, <4 x i32> %val) {
-; CHECK-LABEL: truncstore_v4i32tov4i8:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r0, [r0]
-; CHECK-NEXT:    vmovn.i32 d16, q8
-; CHECK-NEXT:    vuzp.8 d16, d17
-; CHECK-NEXT:    vst1.32 {d16[0]}, [r0:32]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-        %trunc = trunc <4 x i32> %val to <4 x i8>
-	store  <4 x i8> %trunc, ptr %A, align 4
-	ret void
+; CHECK-LE-LABEL: truncstore_v4i32tov4i8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vmovn.i32 d16, q8
+; CHECK-LE-NEXT:    vuzp.8 d16, d17
+; CHECK-LE-NEXT:    vst1.32 {d16[0]}, [r0:32]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: truncstore_v4i32tov4i8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vmovn.i32 d16, q8
+; CHECK-BE-NEXT:    vrev16.8 d16, d16
+; CHECK-BE-NEXT:    vuzp.8 d16, d17
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vrev32.8 d16, d17
+; CHECK-BE-NEXT:    vst1.32 {d16[0]}, [r0:32]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    %trunc = trunc <4 x i32> %val to <4 x i8>
+    store  <4 x i8> %trunc, ptr %A, align 4
+    ret void
 }
 
 define void @truncstore_v4i32tov4i8_fake_update(ptr %ptr, <4 x i32> %val) {
-; CHECK-LABEL: truncstore_v4i32tov4i8_fake_update:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr d17, [sp]
-; CHECK-NEXT:    vmov d16, r2, r3
-; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    movs r2, #16
-; CHECK-NEXT:    vmovn.i32 d16, q8
-; CHECK-NEXT:    vuzp.8 d16, d17
-; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32], r2
-; CHECK-NEXT:    str r1, [r0]
-; CHECK-NEXT:    bx lr
-	%A = load ptr, ptr %ptr
-        %trunc = trunc <4 x i32> %val to <4 x i8>
-	store  <4 x i8> %trunc, ptr %A, align 4
-	%inc = getelementptr <4 x i8>, ptr %A, i38 4
-        store ptr %inc, ptr %ptr
-	ret void
+; CHECK-LE-LABEL: truncstore_v4i32tov4i8_fake_update:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vldr d17, [sp]
+; CHECK-LE-NEXT:    vmov d16, r2, r3
+; CHECK-LE-NEXT:    ldr r1, [r0]
+; CHECK-LE-NEXT:    movs r2, #16
+; CHECK-LE-NEXT:    vmovn.i32 d16, q8
+; CHECK-LE-NEXT:    vuzp.8 d16, d17
+; CHECK-LE-NEXT:    vst1.32 {d16[0]}, [r1:32], r2
+; CHECK-LE-NEXT:    str r1, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: truncstore_v4i32tov4i8_fake_update:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d17, [sp]
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    movs r2, #16
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vmovn.i32 d16, q8
+; CHECK-BE-NEXT:    vrev16.8 d16, d16
+; CHECK-BE-NEXT:    vuzp.8 d16, d17
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    vrev32.8 d16, d17
+; CHECK-BE-NEXT:    vst1.32 {d16[0]}, [r1:32], r2
+; CHECK-BE-NEXT:    str r1, [r0]
+; CHECK-BE-NEXT:    bx lr
+    %A = load ptr, ptr %ptr
+    %trunc = trunc <4 x i32> %val to <4 x i8>
+    store  <4 x i8> %trunc, ptr %A, align 4
+    %inc = getelementptr <4 x i8>, ptr %A, i38 4
+    store ptr %inc, ptr %ptr
+    ret void
 }
 
 define ptr @test_vst1_1reg(ptr %ptr.in, ptr %ptr.out) {
-; CHECK-LABEL: test_vst1_1reg:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    movs r0, #32
-; CHECK-NEXT:    vst1.32 {d16, d17}, [r1], r0
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: test_vst1_1reg:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    movs r0, #32
+; CHECK-LE-NEXT:    vst1.32 {d16, d17}, [r1], r0
+; CHECK-LE-NEXT:    mov r0, r1
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_vst1_1reg:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    movs r0, #32
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vst1.32 {d16, d17}, [r1], r0
+; CHECK-BE-NEXT:    mov r0, r1
+; CHECK-BE-NEXT:    bx lr
   %val = load <4 x i32>, ptr %ptr.in
   store <4 x i32> %val, ptr %ptr.out
   %next = getelementptr <4 x i32>, ptr %ptr.out, i32 2
@@ -400,37 +613,65 @@ define ptr @test_vst1_1reg(ptr %ptr.in, ptr %ptr.out) {
 
 ; PR56970
 define void @v3i8store(ptr %p) {
-; CHECK-LABEL: v3i8store:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    vmov.i32 d16, #0xff
-; CHECK-NEXT:    mov r1, sp
-; CHECK-NEXT:    vmov.i32 d17, #0x0
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vand d16, d17, d16
-; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32]
-; CHECK-NEXT:    vld1.32 {d16[0]}, [r1:32]
-; CHECK-NEXT:    vmovl.u16 q8, d16
-; CHECK-NEXT:    strb r2, [r0, #2]
-; CHECK-NEXT:    vmov.32 r1, d16[0]
-; CHECK-NEXT:    strh r1, [r0]
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: v3i8store:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    mov r2, sp
+; CHECK-LE-NEXT:    str r1, [sp]
+; CHECK-LE-NEXT:    vld1.32 {d16[0]}, [r2:32]
+; CHECK-LE-NEXT:    strb r1, [r0, #2]
+; CHECK-LE-NEXT:    vmovl.u16 q8, d16
+; CHECK-LE-NEXT:    vmov.32 r2, d16[0]
+; CHECK-LE-NEXT:    strh r2, [r0]
+; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: v3i8store:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    mov r2, sp
+; CHECK-BE-NEXT:    str r1, [sp]
+; CHECK-BE-NEXT:    vld1.32 {d16[0]}, [r2:32]
+; CHECK-BE-NEXT:    strb r1, [r0, #2]
+; CHECK-BE-NEXT:    vrev32.16 d16, d16
+; CHECK-BE-NEXT:    vmovl.u16 q8, d16
+; CHECK-BE-NEXT:    vmov.32 r2, d16[0]
+; CHECK-BE-NEXT:    strh r2, [r0]
+; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    bx lr
   store <3 x i8> zeroinitializer, ptr %p, align 4
   ret void
 }
 
 define void @v3i64shuffle(ptr %p, <3 x i64> %a) {
-; CHECK-LABEL: v3i64shuffle:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.i32 q8, #0x0
-; CHECK-NEXT:    ldrd r12, r1, [sp, #8]
-; CHECK-NEXT:    vmov d18, r2, r3
-; CHECK-NEXT:    vorr d19, d16, d16
-; CHECK-NEXT:    str r1, [r0, #20]
-; CHECK-NEXT:    vst1.32 {d18, d19}, [r0]!
-; CHECK-NEXT:    str.w r12, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-LE-LABEL: v3i64shuffle:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vmov.i32 q8, #0x0
+; CHECK-LE-NEXT:    ldrd r12, r1, [sp, #8]
+; CHECK-LE-NEXT:    vmov d18, r2, r3
+; CHECK-LE-NEXT:    vorr d19, d16, d16
+; CHECK-LE-NEXT:    str r1, [r0, #20]
+; CHECK-LE-NEXT:    vst1.32 {d18, d19}, [r0]!
+; CHECK-LE-NEXT:    str.w r12, [r0]
+; CHECK-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: v3i64shuffle:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vldr d16, [sp, #8]
+; CHECK-BE-NEXT:    vmov.i32 q9, #0x0
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vmov r12, r1, d16
+; CHECK-BE-NEXT:    vmov d16, r3, r2
+; CHECK-BE-NEXT:    vorr d17, d18, d18
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    str r1, [r0, #20]
+; CHECK-BE-NEXT:    vst1.32 {d16, d17}, [r0]!
+; CHECK-BE-NEXT:    str.w r12, [r0]
+; CHECK-BE-NEXT:    bx lr
   %b = shufflevector <3 x i64> %a, <3 x i64> zeroinitializer, <3 x i32> <i32 0, i32 3, i32 2>
   store <3 x i64> %b, ptr %p, align 4
   ret void
diff --git a/llvm/test/CodeGen/LoongArch/fp16-promote.ll b/llvm/test/CodeGen/LoongArch/fp16-promote.ll
new file mode 100644
index 0000000000000000000000000000000000000000..75f920b43a06cefedcdb9596e32f76660eed20c0
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/fp16-promote.ll
@@ -0,0 +1,326 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch32 --mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 --mattr=+d < %s | FileCheck %s --check-prefix=LA64
+
+define void @test_load_store(ptr %p, ptr %q) nounwind {
+; LA32-LABEL: test_load_store:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.h $a0, $a0, 0
+; LA32-NEXT:    st.h $a0, $a1, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_load_store:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.h $a0, $a0, 0
+; LA64-NEXT:    st.h $a0, $a1, 0
+; LA64-NEXT:    ret
+  %a = load half, ptr %p
+  store half %a, ptr %q
+  ret void
+}
+
+define float @test_fpextend_float(ptr %p) nounwind {
+; LA32-LABEL: test_fpextend_float:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    b %plt(__gnu_h2f_ieee)
+;
+; LA64-LABEL: test_fpextend_float:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    b %plt(__gnu_h2f_ieee)
+  %a = load half, ptr %p
+  %r = fpext half %a to float
+  ret float %r
+}
+
+define double @test_fpextend_double(ptr %p) nounwind {
+; LA32-LABEL: test_fpextend_double:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fcvt.d.s $fa0, $fa0
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fpextend_double:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  %a = load half, ptr %p
+  %r = fpext half %a to double
+  ret double %r
+}
+
+define void @test_fptrunc_float(float %f, ptr %p) nounwind {
+; LA32-LABEL: test_fptrunc_float:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fptrunc_float:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  %a = fptrunc float %f to half
+  store half %a, ptr %p
+  ret void
+}
+
+define void @test_fptrunc_double(double %d, ptr %p) nounwind {
+; LA32-LABEL: test_fptrunc_double:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl %plt(__truncdfhf2)
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fptrunc_double:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    bl %plt(__truncdfhf2)
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  %a = fptrunc double %d to half
+  store half %a, ptr %p
+  ret void
+}
+
+define half @test_fadd_reg(half %a, half %b) nounwind {
+; LA32-LABEL: test_fadd_reg:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fa1
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fadd.s $fa0, $fa0, $fs1
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fadd_reg:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fa1
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fadd.s $fa0, $fa0, $fs1
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %r = fadd half %a, %b
+  ret half %r
+}
+
+define void @test_fadd_mem(ptr %p, ptr %q) nounwind {
+; LA32-LABEL: test_fadd_mem:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    move $fp, $a1
+; LA32-NEXT:    move $s0, $a0
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    ld.hu $a0, $fp, 0
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fadd.s $fa0, $fs0, $fa0
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    st.h $a0, $s0, 0
+; LA32-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fadd_mem:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a1
+; LA64-NEXT:    move $s0, $a0
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    ld.hu $a0, $fp, 0
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fadd.s $fa0, $fs0, $fa0
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    st.h $a0, $s0, 0
+; LA64-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %a = load half, ptr %p
+  %b = load half, ptr %q
+  %r = fadd half %a, %b
+  store half %r, ptr %p
+  ret void
+}
+
+define half @test_fmul_reg(half %a, half %b) nounwind {
+; LA32-LABEL: test_fmul_reg:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fa1
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmul.s $fa0, $fa0, $fs1
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fmul_reg:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fa1
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmul.s $fa0, $fa0, $fs1
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %r = fmul half %a, %b
+  ret half %r
+}
+
+define void @test_fmul_mem(ptr %p, ptr %q) nounwind {
+; LA32-LABEL: test_fmul_mem:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    move $fp, $a1
+; LA32-NEXT:    move $s0, $a0
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    ld.hu $a0, $fp, 0
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    fmul.s $fa0, $fs0, $fa0
+; LA32-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA32-NEXT:    st.h $a0, $s0, 0
+; LA32-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_fmul_mem:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a1
+; LA64-NEXT:    move $s0, $a0
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    ld.hu $a0, $fp, 0
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    fmul.s $fa0, $fs0, $fa0
+; LA64-NEXT:    bl %plt(__gnu_f2h_ieee)
+; LA64-NEXT:    st.h $a0, $s0, 0
+; LA64-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %a = load half, ptr %p
+  %b = load half, ptr %q
+  %r = fmul half %a, %b
+  store half %r, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/global-address.ll b/llvm/test/CodeGen/LoongArch/global-address.ll
index 0c8958b6ab336ab51b2d9bd111281d4d10f6f946..fb2929572f31aabcf2a871926e8bb5294c6793e6 100644
--- a/llvm/test/CodeGen/LoongArch/global-address.ll
+++ b/llvm/test/CodeGen/LoongArch/global-address.ll
@@ -14,40 +14,40 @@ define void @foo() nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; LA32NOPIC-NEXT:    ld.w $a0, $a0, %got_pc_lo12(G)
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32NOPIC-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; LA32NOPIC-NEXT:    addi.w $a0, $a0, %pc_lo12(g)
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32NOPIC-NEXT:    ret
 ;
 ; LA32PIC-LABEL: foo:
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; LA32PIC-NEXT:    ld.w $a0, $a0, %got_pc_lo12(G)
-; LA32PIC-NEXT:    ld.w $a0, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32PIC-NEXT:    pcalau12i $a0, %pc_hi20(.Lg$local)
 ; LA32PIC-NEXT:    addi.w $a0, $a0, %pc_lo12(.Lg$local)
-; LA32PIC-NEXT:    ld.w $a0, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32PIC-NEXT:    ret
 ;
 ; LA64NOPIC-LABEL: foo:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; LA64NOPIC-NEXT:    ld.d $a0, $a0, %got_pc_lo12(G)
-; LA64NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64NOPIC-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; LA64NOPIC-NEXT:    addi.d $a0, $a0, %pc_lo12(g)
-; LA64NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: foo:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; LA64PIC-NEXT:    ld.d $a0, $a0, %got_pc_lo12(G)
-; LA64PIC-NEXT:    ld.w $a0, $a0, 0
+; LA64PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64PIC-NEXT:    pcalau12i $a0, %pc_hi20(.Lg$local)
 ; LA64PIC-NEXT:    addi.d $a0, $a0, %pc_lo12(.Lg$local)
-; LA64PIC-NEXT:    ld.w $a0, $a0, 0
+; LA64PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64PIC-NEXT:    ret
 ;
 ; LA64LARGENOPIC-LABEL: foo:
@@ -57,13 +57,13 @@ define void @foo() nounwind {
 ; LA64LARGENOPIC-NEXT:    lu32i.d $t8, %got64_pc_lo20(G)
 ; LA64LARGENOPIC-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(G)
 ; LA64LARGENOPIC-NEXT:    ldx.d $a0, $t8, $a0
-; LA64LARGENOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64LARGENOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64LARGENOPIC-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; LA64LARGENOPIC-NEXT:    addi.d $t8, $zero, %pc_lo12(g)
 ; LA64LARGENOPIC-NEXT:    lu32i.d $t8, %pc64_lo20(g)
 ; LA64LARGENOPIC-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(g)
 ; LA64LARGENOPIC-NEXT:    add.d $a0, $t8, $a0
-; LA64LARGENOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64LARGENOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64LARGENOPIC-NEXT:    ret
 ;
 ; LA64LARGEPIC-LABEL: foo:
@@ -73,13 +73,13 @@ define void @foo() nounwind {
 ; LA64LARGEPIC-NEXT:    lu32i.d $t8, %got64_pc_lo20(G)
 ; LA64LARGEPIC-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(G)
 ; LA64LARGEPIC-NEXT:    ldx.d $a0, $t8, $a0
-; LA64LARGEPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64LARGEPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64LARGEPIC-NEXT:    pcalau12i $a0, %pc_hi20(.Lg$local)
 ; LA64LARGEPIC-NEXT:    addi.d $t8, $zero, %pc_lo12(.Lg$local)
 ; LA64LARGEPIC-NEXT:    lu32i.d $t8, %pc64_lo20(.Lg$local)
 ; LA64LARGEPIC-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(.Lg$local)
 ; LA64LARGEPIC-NEXT:    add.d $a0, $t8, $a0
-; LA64LARGEPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64LARGEPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64LARGEPIC-NEXT:    ret
   %V = load volatile i32, ptr @G
   %v = load volatile i32, ptr @g
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
index 4a59e2af533e76c28bfbda3b5150da069436939c..d94da34e14c4c8a797b4147ef8e7d6ac5451fb59 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
@@ -178,7 +178,7 @@ entry:
 define void @csrrd_d_noret() {
 ; CHECK-LABEL: csrrd_d_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrrd $a0, 1
+; CHECK-NEXT:    csrrd $zero, 1
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.loongarch.csrrd.d(i32 1)
@@ -240,7 +240,7 @@ entry:
 define void @iocsrrd_d_noret(i32 %a) {
 ; CHECK-LABEL: iocsrrd_d_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    iocsrrd.d $a0, $a0
+; CHECK-NEXT:    iocsrrd.d $zero, $a0
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.loongarch.iocsrrd.d(i32 %a)
@@ -290,7 +290,7 @@ entry:
 define void @lddir_d_noret(i64 %a) {
 ; CHECK-LABEL: lddir_d_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lddir $a0, $a0, 1
+; CHECK-NEXT:    lddir $zero, $a0, 1
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i64 @llvm.loongarch.lddir.d(i64 %a, i64 1)
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic.ll b/llvm/test/CodeGen/LoongArch/intrinsic.ll
index f49a2500ad3c73137a4e4a1555479cab2834a4cd..6d8adb5da92615505f50d27ab5464c1074fbac6f 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic.ll
@@ -73,7 +73,7 @@ entry:
 define void @movfcsr2gr_noret() nounwind {
 ; CHECK-LABEL: movfcsr2gr_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movfcsr2gr $a0, $fcsr1
+; CHECK-NEXT:    movfcsr2gr $zero, $fcsr1
 ; CHECK-NEXT:    ret
 entry:
   %res = call i32 @llvm.loongarch.movfcsr2gr(i32 1)
@@ -103,7 +103,7 @@ entry:
 define void @csrrd_w_noret() {
 ; CHECK-LABEL: csrrd_w_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrrd $a0, 1
+; CHECK-NEXT:    csrrd $zero, 1
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i32 @llvm.loongarch.csrrd.w(i32 1)
@@ -185,7 +185,7 @@ entry:
 define void @iocsrrd_b_noret(i32 %a) {
 ; CHECK-LABEL: iocsrrd_b_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    iocsrrd.b $a0, $a0
+; CHECK-NEXT:    iocsrrd.b $zero, $a0
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i32 @llvm.loongarch.iocsrrd.b(i32 %a)
@@ -195,7 +195,7 @@ entry:
 define void @iocsrrd_h_noret(i32 %a) {
 ; CHECK-LABEL: iocsrrd_h_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    iocsrrd.h $a0, $a0
+; CHECK-NEXT:    iocsrrd.h $zero, $a0
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i32 @llvm.loongarch.iocsrrd.h(i32 %a)
@@ -205,7 +205,7 @@ entry:
 define void @iocsrrd_w_noret(i32 %a) {
 ; CHECK-LABEL: iocsrrd_w_noret:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    iocsrrd.w $a0, $a0
+; CHECK-NEXT:    iocsrrd.w $zero, $a0
 ; CHECK-NEXT:    ret
 entry:
   %0 = tail call i32 @llvm.loongarch.iocsrrd.w(i32 %a)
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
index 02375a925723d28fab193743d0cdd5a676bc25f9..8b909e3314d642aa928f40a7e12f4bca71a371de 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
@@ -21,7 +21,7 @@ define void @foo_br_eq(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    beq $a2, $a0, .LBB1_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB1_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -31,7 +31,7 @@ define void @foo_br_eq(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    beq $a2, $a0, .LBB1_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB1_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -51,7 +51,7 @@ define void @foo_br_ne(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bne $a2, $a0, .LBB2_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB2_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -61,7 +61,7 @@ define void @foo_br_ne(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bne $a2, $a0, .LBB2_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB2_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -81,7 +81,7 @@ define void @foo_br_slt(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    blt $a2, $a0, .LBB3_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB3_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -91,7 +91,7 @@ define void @foo_br_slt(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    blt $a2, $a0, .LBB3_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB3_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -111,7 +111,7 @@ define void @foo_br_sge(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bge $a2, $a0, .LBB4_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB4_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -121,7 +121,7 @@ define void @foo_br_sge(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bge $a2, $a0, .LBB4_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB4_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -141,7 +141,7 @@ define void @foo_br_ult(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bltu $a2, $a0, .LBB5_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB5_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -151,7 +151,7 @@ define void @foo_br_ult(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bltu $a2, $a0, .LBB5_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB5_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -171,7 +171,7 @@ define void @foo_br_uge(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bgeu $a2, $a0, .LBB6_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB6_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -181,7 +181,7 @@ define void @foo_br_uge(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bgeu $a2, $a0, .LBB6_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB6_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -202,7 +202,7 @@ define void @foo_br_sgt(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    blt $a0, $a2, .LBB7_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB7_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -212,7 +212,7 @@ define void @foo_br_sgt(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    blt $a0, $a2, .LBB7_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB7_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -232,7 +232,7 @@ define void @foo_br_sle(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bge $a0, $a2, .LBB8_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB8_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -242,7 +242,7 @@ define void @foo_br_sle(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bge $a0, $a2, .LBB8_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB8_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -262,7 +262,7 @@ define void @foo_br_ugt(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bltu $a0, $a2, .LBB9_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB9_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -272,7 +272,7 @@ define void @foo_br_ugt(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bltu $a0, $a2, .LBB9_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB9_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -292,7 +292,7 @@ define void @foo_br_ule(i32 %a, ptr %b) nounwind {
 ; LA32-NEXT:    ld.w $a2, $a1, 0
 ; LA32-NEXT:    bgeu $a0, $a2, .LBB10_2
 ; LA32-NEXT:  # %bb.1: # %test
-; LA32-NEXT:    ld.w $a0, $a1, 0
+; LA32-NEXT:    ld.w $zero, $a1, 0
 ; LA32-NEXT:  .LBB10_2: # %end
 ; LA32-NEXT:    ret
 ;
@@ -302,7 +302,7 @@ define void @foo_br_ule(i32 %a, ptr %b) nounwind {
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bgeu $a0, $a2, .LBB10_2
 ; LA64-NEXT:  # %bb.1: # %test
-; LA64-NEXT:    ld.w $a0, $a1, 0
+; LA64-NEXT:    ld.w $zero, $a1, 0
 ; LA64-NEXT:  .LBB10_2: # %end
 ; LA64-NEXT:    ret
   %val = load volatile i32, ptr %b
@@ -321,11 +321,11 @@ end:
 define void @foo_br_cc(ptr %a, i1 %cc) nounwind {
 ; ALL-LABEL: foo_br_cc:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    ld.w $a2, $a0, 0
+; ALL-NEXT:    ld.w $zero, $a0, 0
 ; ALL-NEXT:    andi $a1, $a1, 1
 ; ALL-NEXT:    bnez $a1, .LBB11_2
 ; ALL-NEXT:  # %bb.1: # %test
-; ALL-NEXT:    ld.w $a0, $a0, 0
+; ALL-NEXT:    ld.w $zero, $a0, 0
 ; ALL-NEXT:  .LBB11_2: # %end
 ; ALL-NEXT:    ret
   %val = load volatile i32, ptr %a
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll
index 387d9333b90f0534f88315d1345389198c7295ee..26f44adc6135865ad90ba295b6ee3b000ded9004 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll
@@ -57,7 +57,7 @@ define i32 @load_store_global_array(i32 %a) nounwind {
 ; LA32NOPIC-NEXT:    addi.w $a2, $a1, %pc_lo12(arr)
 ; LA32NOPIC-NEXT:    ld.w $a1, $a2, 0
 ; LA32NOPIC-NEXT:    st.w $a0, $a2, 0
-; LA32NOPIC-NEXT:    ld.w $a3, $a2, 36
+; LA32NOPIC-NEXT:    ld.w $zero, $a2, 36
 ; LA32NOPIC-NEXT:    st.w $a0, $a2, 36
 ; LA32NOPIC-NEXT:    move $a0, $a1
 ; LA32NOPIC-NEXT:    ret
@@ -68,7 +68,7 @@ define i32 @load_store_global_array(i32 %a) nounwind {
 ; LA32PIC-NEXT:    addi.w $a2, $a1, %pc_lo12(.Larr$local)
 ; LA32PIC-NEXT:    ld.w $a1, $a2, 0
 ; LA32PIC-NEXT:    st.w $a0, $a2, 0
-; LA32PIC-NEXT:    ld.w $a3, $a2, 36
+; LA32PIC-NEXT:    ld.w $zero, $a2, 36
 ; LA32PIC-NEXT:    st.w $a0, $a2, 36
 ; LA32PIC-NEXT:    move $a0, $a1
 ; LA32PIC-NEXT:    ret
@@ -79,7 +79,7 @@ define i32 @load_store_global_array(i32 %a) nounwind {
 ; LA64NOPIC-NEXT:    addi.d $a2, $a1, %pc_lo12(arr)
 ; LA64NOPIC-NEXT:    ld.w $a1, $a2, 0
 ; LA64NOPIC-NEXT:    st.w $a0, $a2, 0
-; LA64NOPIC-NEXT:    ld.w $a3, $a2, 36
+; LA64NOPIC-NEXT:    ld.w $zero, $a2, 36
 ; LA64NOPIC-NEXT:    st.w $a0, $a2, 36
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
@@ -90,7 +90,7 @@ define i32 @load_store_global_array(i32 %a) nounwind {
 ; LA64PIC-NEXT:    addi.d $a2, $a1, %pc_lo12(.Larr$local)
 ; LA64PIC-NEXT:    ld.w $a1, $a2, 0
 ; LA64PIC-NEXT:    st.w $a0, $a2, 0
-; LA64PIC-NEXT:    ld.w $a3, $a2, 36
+; LA64PIC-NEXT:    ld.w $zero, $a2, 36
 ; LA64PIC-NEXT:    st.w $a0, $a2, 36
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
@@ -108,7 +108,7 @@ define i64 @ld_b(ptr %a) nounwind {
 ; LA32NOPIC-LABEL: ld_b:
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    ld.b $a2, $a0, 1
-; LA32NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -116,7 +116,7 @@ define i64 @ld_b(ptr %a) nounwind {
 ; LA32PIC-LABEL: ld_b:
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    ld.b $a2, $a0, 1
-; LA32PIC-NEXT:    ld.b $a0, $a0, 0
+; LA32PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -124,14 +124,14 @@ define i64 @ld_b(ptr %a) nounwind {
 ; LA64NOPIC-LABEL: ld_b:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.b $a1, $a0, 1
-; LA64NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: ld_b:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.b $a1, $a0, 1
-; LA64PIC-NEXT:    ld.b $a0, $a0, 0
+; LA64PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i8, ptr %a, i64 1
@@ -145,7 +145,7 @@ define i64 @ld_h(ptr %a) nounwind {
 ; LA32NOPIC-LABEL: ld_h:
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    ld.h $a2, $a0, 4
-; LA32NOPIC-NEXT:    ld.h $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.h $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -153,7 +153,7 @@ define i64 @ld_h(ptr %a) nounwind {
 ; LA32PIC-LABEL: ld_h:
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    ld.h $a2, $a0, 4
-; LA32PIC-NEXT:    ld.h $a0, $a0, 0
+; LA32PIC-NEXT:    ld.h $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -161,14 +161,14 @@ define i64 @ld_h(ptr %a) nounwind {
 ; LA64NOPIC-LABEL: ld_h:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.h $a1, $a0, 4
-; LA64NOPIC-NEXT:    ld.h $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.h $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: ld_h:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.h $a1, $a0, 4
-; LA64PIC-NEXT:    ld.h $a0, $a0, 0
+; LA64PIC-NEXT:    ld.h $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i16, ptr %a, i64 2
@@ -182,7 +182,7 @@ define i64 @ld_w(ptr %a) nounwind {
 ; LA32NOPIC-LABEL: ld_w:
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    ld.w $a2, $a0, 12
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -190,7 +190,7 @@ define i64 @ld_w(ptr %a) nounwind {
 ; LA32PIC-LABEL: ld_w:
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    ld.w $a2, $a0, 12
-; LA32PIC-NEXT:    ld.w $a0, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -198,14 +198,14 @@ define i64 @ld_w(ptr %a) nounwind {
 ; LA64NOPIC-LABEL: ld_w:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.w $a1, $a0, 12
-; LA64NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: ld_w:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.w $a1, $a0, 12
-; LA64PIC-NEXT:    ld.w $a0, $a0, 0
+; LA64PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i32, ptr %a, i64 3
@@ -220,8 +220,8 @@ define i64 @ld_d(ptr %a) nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    ld.w $a1, $a0, 28
 ; LA32NOPIC-NEXT:    ld.w $a2, $a0, 24
-; LA32NOPIC-NEXT:    ld.w $a3, $a0, 4
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 4
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
 ;
@@ -229,22 +229,22 @@ define i64 @ld_d(ptr %a) nounwind {
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    ld.w $a1, $a0, 28
 ; LA32PIC-NEXT:    ld.w $a2, $a0, 24
-; LA32PIC-NEXT:    ld.w $a3, $a0, 4
-; LA32PIC-NEXT:    ld.w $a0, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 4
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
 ;
 ; LA64NOPIC-LABEL: ld_d:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.d $a1, $a0, 24
-; LA64NOPIC-NEXT:    ld.d $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.d $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: ld_d:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.d $a1, $a0, 24
-; LA64PIC-NEXT:    ld.d $a0, $a0, 0
+; LA64PIC-NEXT:    ld.d $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i64, ptr %a, i64 3
@@ -375,7 +375,7 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    add.w $a1, $a0, $a1
 ; LA32NOPIC-NEXT:    ld.b $a2, $a1, 0
-; LA32NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -384,7 +384,7 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind {
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    add.w $a1, $a0, $a1
 ; LA32PIC-NEXT:    ld.b $a2, $a1, 0
-; LA32PIC-NEXT:    ld.b $a0, $a0, 0
+; LA32PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -392,14 +392,14 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind {
 ; LA64NOPIC-LABEL: ldx_b:
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ldx.b $a1, $a0, $a1
-; LA64NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
 ; LA64PIC-LABEL: ldx_b:
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ldx.b $a1, $a0, $a1
-; LA64PIC-NEXT:    ld.b $a0, $a0, 0
+; LA64PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i8, ptr %a, i64 %idx
@@ -414,7 +414,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    alsl.w $a1, $a1, $a0, 1
 ; LA32NOPIC-NEXT:    ld.h $a2, $a1, 0
-; LA32NOPIC-NEXT:    ld.h $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.h $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -423,7 +423,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind {
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    alsl.w $a1, $a1, $a0, 1
 ; LA32PIC-NEXT:    ld.h $a2, $a1, 0
-; LA32PIC-NEXT:    ld.h $a0, $a0, 0
+; LA32PIC-NEXT:    ld.h $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -432,7 +432,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind {
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    slli.d $a1, $a1, 1
 ; LA64NOPIC-NEXT:    ldx.h $a1, $a0, $a1
-; LA64NOPIC-NEXT:    ld.h $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.h $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
@@ -440,7 +440,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind {
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    slli.d $a1, $a1, 1
 ; LA64PIC-NEXT:    ldx.h $a1, $a0, $a1
-; LA64PIC-NEXT:    ld.h $a0, $a0, 0
+; LA64PIC-NEXT:    ld.h $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i16, ptr %a, i64 %idx
@@ -455,7 +455,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    alsl.w $a1, $a1, $a0, 2
 ; LA32NOPIC-NEXT:    ld.w $a2, $a1, 0
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32NOPIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
@@ -464,7 +464,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind {
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    alsl.w $a1, $a1, $a0, 2
 ; LA32PIC-NEXT:    ld.w $a2, $a1, 0
-; LA32PIC-NEXT:    ld.w $a0, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA32PIC-NEXT:    srai.w $a1, $a2, 31
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
@@ -473,7 +473,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind {
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    slli.d $a1, $a1, 2
 ; LA64NOPIC-NEXT:    ldx.w $a1, $a0, $a1
-; LA64NOPIC-NEXT:    ld.w $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
@@ -481,7 +481,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind {
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    slli.d $a1, $a1, 2
 ; LA64PIC-NEXT:    ldx.w $a1, $a0, $a1
-; LA64PIC-NEXT:    ld.w $a0, $a0, 0
+; LA64PIC-NEXT:    ld.w $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i32, ptr %a, i64 %idx
@@ -497,8 +497,8 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind {
 ; LA32NOPIC-NEXT:    alsl.w $a1, $a1, $a0, 3
 ; LA32NOPIC-NEXT:    ld.w $a2, $a1, 0
 ; LA32NOPIC-NEXT:    ld.w $a1, $a1, 4
-; LA32NOPIC-NEXT:    ld.w $a3, $a0, 0
-; LA32NOPIC-NEXT:    ld.w $a0, $a0, 4
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 0
+; LA32NOPIC-NEXT:    ld.w $zero, $a0, 4
 ; LA32NOPIC-NEXT:    move $a0, $a2
 ; LA32NOPIC-NEXT:    ret
 ;
@@ -507,8 +507,8 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind {
 ; LA32PIC-NEXT:    alsl.w $a1, $a1, $a0, 3
 ; LA32PIC-NEXT:    ld.w $a2, $a1, 0
 ; LA32PIC-NEXT:    ld.w $a1, $a1, 4
-; LA32PIC-NEXT:    ld.w $a3, $a0, 0
-; LA32PIC-NEXT:    ld.w $a0, $a0, 4
+; LA32PIC-NEXT:    ld.w $zero, $a0, 0
+; LA32PIC-NEXT:    ld.w $zero, $a0, 4
 ; LA32PIC-NEXT:    move $a0, $a2
 ; LA32PIC-NEXT:    ret
 ;
@@ -516,7 +516,7 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind {
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    slli.d $a1, $a1, 3
 ; LA64NOPIC-NEXT:    ldx.d $a1, $a0, $a1
-; LA64NOPIC-NEXT:    ld.d $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.d $zero, $a0, 0
 ; LA64NOPIC-NEXT:    move $a0, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
@@ -524,7 +524,7 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind {
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    slli.d $a1, $a1, 3
 ; LA64PIC-NEXT:    ldx.d $a1, $a0, $a1
-; LA64PIC-NEXT:    ld.d $a0, $a0, 0
+; LA64PIC-NEXT:    ld.d $zero, $a0, 0
 ; LA64PIC-NEXT:    move $a0, $a1
 ; LA64PIC-NEXT:    ret
   %1 = getelementptr i64, ptr %a, i64 %idx
@@ -855,7 +855,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind {
 ; LA32NOPIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA32NOPIC-NEXT:    ld.bu $a3, $a0, 2
 ; LA32NOPIC-NEXT:    sub.w $a2, $a3, $a1
-; LA32NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32NOPIC-NEXT:    sltu $a0, $a3, $a1
 ; LA32NOPIC-NEXT:    sub.w $a1, $zero, $a0
 ; LA32NOPIC-NEXT:    move $a0, $a2
@@ -866,7 +866,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind {
 ; LA32PIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA32PIC-NEXT:    ld.bu $a3, $a0, 2
 ; LA32PIC-NEXT:    sub.w $a2, $a3, $a1
-; LA32PIC-NEXT:    ld.b $a0, $a0, 0
+; LA32PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32PIC-NEXT:    sltu $a0, $a3, $a1
 ; LA32PIC-NEXT:    sub.w $a1, $zero, $a0
 ; LA32PIC-NEXT:    move $a0, $a2
@@ -876,7 +876,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind {
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA64NOPIC-NEXT:    ld.bu $a2, $a0, 2
-; LA64NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64NOPIC-NEXT:    sub.d $a0, $a2, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
@@ -884,7 +884,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind {
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA64PIC-NEXT:    ld.bu $a2, $a0, 2
-; LA64PIC-NEXT:    ld.b $a0, $a0, 0
+; LA64PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64PIC-NEXT:    sub.d $a0, $a2, $a1
 ; LA64PIC-NEXT:    ret
   ;; sextload i1
@@ -906,7 +906,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind {
 ; LA32NOPIC:       # %bb.0:
 ; LA32NOPIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA32NOPIC-NEXT:    ld.bu $a2, $a0, 2
-; LA32NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA32NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32NOPIC-NEXT:    sub.w $a0, $a2, $a1
 ; LA32NOPIC-NEXT:    ret
 ;
@@ -914,7 +914,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind {
 ; LA32PIC:       # %bb.0:
 ; LA32PIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA32PIC-NEXT:    ld.bu $a2, $a0, 2
-; LA32PIC-NEXT:    ld.b $a0, $a0, 0
+; LA32PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA32PIC-NEXT:    sub.w $a0, $a2, $a1
 ; LA32PIC-NEXT:    ret
 ;
@@ -922,7 +922,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind {
 ; LA64NOPIC:       # %bb.0:
 ; LA64NOPIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA64NOPIC-NEXT:    ld.bu $a2, $a0, 2
-; LA64NOPIC-NEXT:    ld.b $a0, $a0, 0
+; LA64NOPIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64NOPIC-NEXT:    sub.d $a0, $a2, $a1
 ; LA64NOPIC-NEXT:    ret
 ;
@@ -930,7 +930,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind {
 ; LA64PIC:       # %bb.0:
 ; LA64PIC-NEXT:    ld.bu $a1, $a0, 1
 ; LA64PIC-NEXT:    ld.bu $a2, $a0, 2
-; LA64PIC-NEXT:    ld.b $a0, $a0, 0
+; LA64PIC-NEXT:    ld.b $zero, $a0, 0
 ; LA64PIC-NEXT:    sub.d $a0, $a2, $a1
 ; LA64PIC-NEXT:    ret
   ;; sextload i1
diff --git a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cd308a1495914ad2779ead2f5015a826274ff001
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll
@@ -0,0 +1,458 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 --relocation-model=pic \
+; RUN:   --verify-machineinstrs < %s | FileCheck -check-prefixes=LA32 %s
+; RUN: llc --mtriple=loongarch64 --relocation-model=pic \
+; RUN:   --verify-machineinstrs < %s | FileCheck -check-prefixes=LA64 %s
+; RUN: llc --mtriple=loongarch64 --relocation-model=pic --code-model=large \
+; RUN:   --verify-machineinstrs < %s | FileCheck -check-prefixes=LA64LARGE %s
+
+; Verifies that MachineLICM can hoist address generation pseudos out of loops.
+
+@l = protected global i32 0, align 4
+
+define void @test_la_pcrel(i32 signext %n) {
+; LA32-LABEL: test_la_pcrel:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    pcalau12i $a2, %pc_hi20(l)
+; LA32-NEXT:    addi.w $a2, $a2, %pc_lo12(l)
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB0_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ld.w $zero, $a2, 0
+; LA32-NEXT:    addi.w $a1, $a1, 1
+; LA32-NEXT:    blt $a1, $a0, .LBB0_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_pcrel:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    move $a1, $zero
+; LA64-NEXT:    pcalau12i $a2, %pc_hi20(l)
+; LA64-NEXT:    addi.d $a2, $a2, %pc_lo12(l)
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB0_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ld.w $zero, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 1
+; LA64-NEXT:    blt $a1, $a0, .LBB0_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_pcrel:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    move $a1, $zero
+; LA64LARGE-NEXT:    pcalau12i $a2, %pc_hi20(l)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %pc_lo12(l)
+; LA64LARGE-NEXT:    lu32i.d $t8, %pc64_lo20(l)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(l)
+; LA64LARGE-NEXT:    add.d $a2, $t8, $a2
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB0_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    ld.w $zero, $a2, 0
+; LA64LARGE-NEXT:    addi.w $a1, $a1, 1
+; LA64LARGE-NEXT:    blt $a1, $a0, .LBB0_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @l, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+@g = global i32 0, align 4
+
+define void @test_la_got(i32 signext %n) {
+; LA32-LABEL: test_la_got:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB1_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    pcalau12i $a2, %got_pc_hi20(g)
+; LA32-NEXT:    ld.w $a2, $a2, %got_pc_lo12(g)
+; LA32-NEXT:    ld.w $zero, $a2, 0
+; LA32-NEXT:    addi.w $a1, $a1, 1
+; LA32-NEXT:    blt $a1, $a0, .LBB1_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_got:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    move $a1, $zero
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB1_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    pcalau12i $a2, %got_pc_hi20(g)
+; LA64-NEXT:    ld.d $a2, $a2, %got_pc_lo12(g)
+; LA64-NEXT:    ld.w $zero, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 1
+; LA64-NEXT:    blt $a1, $a0, .LBB1_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_got:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    move $a1, $zero
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB1_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    pcalau12i $a2, %got_pc_hi20(g)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %got_pc_lo12(g)
+; LA64LARGE-NEXT:    lu32i.d $t8, %got64_pc_lo20(g)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(g)
+; LA64LARGE-NEXT:    ldx.d $a2, $t8, $a2
+; LA64LARGE-NEXT:    ld.w $zero, $a2, 0
+; LA64LARGE-NEXT:    addi.w $a1, $a1, 1
+; LA64LARGE-NEXT:    blt $a1, $a0, .LBB1_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @g, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+@ie = external thread_local(initialexec) global i32
+
+define void @test_la_tls_ie(i32 signext %n) {
+; LA32-LABEL: test_la_tls_ie:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB2_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    pcalau12i $a2, %ie_pc_hi20(ie)
+; LA32-NEXT:    ld.w $a2, $a2, %ie_pc_lo12(ie)
+; LA32-NEXT:    add.w $a2, $a2, $tp
+; LA32-NEXT:    ld.w $zero, $a2, 0
+; LA32-NEXT:    addi.w $a1, $a1, 1
+; LA32-NEXT:    blt $a1, $a0, .LBB2_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_tls_ie:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    move $a1, $zero
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB2_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    pcalau12i $a2, %ie_pc_hi20(ie)
+; LA64-NEXT:    ld.d $a2, $a2, %ie_pc_lo12(ie)
+; LA64-NEXT:    ldx.w $zero, $a2, $tp
+; LA64-NEXT:    addi.w $a1, $a1, 1
+; LA64-NEXT:    blt $a1, $a0, .LBB2_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_tls_ie:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    move $a1, $zero
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB2_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    pcalau12i $a2, %ie_pc_hi20(ie)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %ie_pc_lo12(ie)
+; LA64LARGE-NEXT:    lu32i.d $t8, %ie64_pc_lo20(ie)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(ie)
+; LA64LARGE-NEXT:    ldx.d $a2, $t8, $a2
+; LA64LARGE-NEXT:    ldx.w $zero, $a2, $tp
+; LA64LARGE-NEXT:    addi.w $a1, $a1, 1
+; LA64LARGE-NEXT:    blt $a1, $a0, .LBB2_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @ie, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+@ld = external thread_local(localdynamic) global i32
+
+define void @test_la_tls_ld(i32 signext %n) {
+; LA32-LABEL: test_la_tls_ld:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    .cfi_def_cfa_offset 16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 4 # 4-byte Folded Spill
+; LA32-NEXT:    .cfi_offset 1, -4
+; LA32-NEXT:    .cfi_offset 22, -8
+; LA32-NEXT:    .cfi_offset 23, -12
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    move $s0, $zero
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB3_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    pcalau12i $a0, %ld_pc_hi20(ld)
+; LA32-NEXT:    addi.w $a0, $a0, %got_pc_lo12(ld)
+; LA32-NEXT:    bl %plt(__tls_get_addr)
+; LA32-NEXT:    ld.w $zero, $a0, 0
+; LA32-NEXT:    addi.w $s0, $s0, 1
+; LA32-NEXT:    blt $s0, $fp, .LBB3_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ld.w $s0, $sp, 4 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_tls_ld:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    .cfi_def_cfa_offset 32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    .cfi_offset 1, -8
+; LA64-NEXT:    .cfi_offset 22, -16
+; LA64-NEXT:    .cfi_offset 23, -24
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    move $s0, $zero
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB3_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    pcalau12i $a0, %ld_pc_hi20(ld)
+; LA64-NEXT:    addi.d $a0, $a0, %got_pc_lo12(ld)
+; LA64-NEXT:    bl %plt(__tls_get_addr)
+; LA64-NEXT:    ld.w $zero, $a0, 0
+; LA64-NEXT:    addi.w $s0, $s0, 1
+; LA64-NEXT:    blt $s0, $fp, .LBB3_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_tls_ld:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    addi.d $sp, $sp, -32
+; LA64LARGE-NEXT:    .cfi_def_cfa_offset 32
+; LA64LARGE-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    .cfi_offset 1, -8
+; LA64LARGE-NEXT:    .cfi_offset 22, -16
+; LA64LARGE-NEXT:    .cfi_offset 23, -24
+; LA64LARGE-NEXT:    move $fp, $a0
+; LA64LARGE-NEXT:    move $s0, $zero
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB3_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    pcalau12i $a0, %ld_pc_hi20(ld)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %got_pc_lo12(ld)
+; LA64LARGE-NEXT:    lu32i.d $t8, %got64_pc_lo20(ld)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(ld)
+; LA64LARGE-NEXT:    add.d $a0, $t8, $a0
+; LA64LARGE-NEXT:    pcalau12i $ra, %pc_hi20(__tls_get_addr)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %pc_lo12(__tls_get_addr)
+; LA64LARGE-NEXT:    lu32i.d $t8, %pc64_lo20(__tls_get_addr)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr)
+; LA64LARGE-NEXT:    add.d $ra, $t8, $ra
+; LA64LARGE-NEXT:    jirl $ra, $ra, 0
+; LA64LARGE-NEXT:    ld.w $zero, $a0, 0
+; LA64LARGE-NEXT:    addi.w $s0, $s0, 1
+; LA64LARGE-NEXT:    blt $s0, $fp, .LBB3_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    addi.d $sp, $sp, 32
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @ld, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+@le = external thread_local(localexec) global i32
+
+define void @test_la_tls_le(i32 signext %n) {
+; LA32-LABEL: test_la_tls_le:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    lu12i.w $a2, %le_hi20(le)
+; LA32-NEXT:    ori $a2, $a2, %le_lo12(le)
+; LA32-NEXT:    add.w $a2, $a2, $tp
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB4_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ld.w $zero, $a2, 0
+; LA32-NEXT:    addi.w $a1, $a1, 1
+; LA32-NEXT:    blt $a1, $a0, .LBB4_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_tls_le:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    move $a1, $zero
+; LA64-NEXT:    lu12i.w $a2, %le_hi20(le)
+; LA64-NEXT:    ori $a2, $a2, %le_lo12(le)
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB4_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ldx.w $zero, $a2, $tp
+; LA64-NEXT:    addi.w $a1, $a1, 1
+; LA64-NEXT:    blt $a1, $a0, .LBB4_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_tls_le:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    move $a1, $zero
+; LA64LARGE-NEXT:    lu12i.w $a2, %le_hi20(le)
+; LA64LARGE-NEXT:    ori $a2, $a2, %le_lo12(le)
+; LA64LARGE-NEXT:    lu32i.d $a2, %le64_lo20(le)
+; LA64LARGE-NEXT:    lu52i.d $a2, $a2, %le64_hi12(le)
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB4_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    ldx.w $zero, $a2, $tp
+; LA64LARGE-NEXT:    addi.w $a1, $a1, 1
+; LA64LARGE-NEXT:    blt $a1, $a0, .LBB4_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @le, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+@gd = external thread_local global i32
+
+define void @test_la_tls_gd(i32 signext %n) nounwind {
+; LA32-LABEL: test_la_tls_gd:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 4 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    move $s0, $zero
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB5_1: # %loop
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    pcalau12i $a0, %gd_pc_hi20(gd)
+; LA32-NEXT:    addi.w $a0, $a0, %got_pc_lo12(gd)
+; LA32-NEXT:    bl %plt(__tls_get_addr)
+; LA32-NEXT:    ld.w $zero, $a0, 0
+; LA32-NEXT:    addi.w $s0, $s0, 1
+; LA32-NEXT:    blt $s0, $fp, .LBB5_1
+; LA32-NEXT:  # %bb.2: # %ret
+; LA32-NEXT:    ld.w $s0, $sp, 4 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_la_tls_gd:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    move $s0, $zero
+; LA64-NEXT:    .p2align 4, , 16
+; LA64-NEXT:  .LBB5_1: # %loop
+; LA64-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    pcalau12i $a0, %gd_pc_hi20(gd)
+; LA64-NEXT:    addi.d $a0, $a0, %got_pc_lo12(gd)
+; LA64-NEXT:    bl %plt(__tls_get_addr)
+; LA64-NEXT:    ld.w $zero, $a0, 0
+; LA64-NEXT:    addi.w $s0, $s0, 1
+; LA64-NEXT:    blt $s0, $fp, .LBB5_1
+; LA64-NEXT:  # %bb.2: # %ret
+; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+;
+; LA64LARGE-LABEL: test_la_tls_gd:
+; LA64LARGE:       # %bb.0: # %entry
+; LA64LARGE-NEXT:    addi.d $sp, $sp, -32
+; LA64LARGE-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64LARGE-NEXT:    move $fp, $a0
+; LA64LARGE-NEXT:    move $s0, $zero
+; LA64LARGE-NEXT:    .p2align 4, , 16
+; LA64LARGE-NEXT:  .LBB5_1: # %loop
+; LA64LARGE-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64LARGE-NEXT:    pcalau12i $a0, %gd_pc_hi20(gd)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %got_pc_lo12(gd)
+; LA64LARGE-NEXT:    lu32i.d $t8, %got64_pc_lo20(gd)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(gd)
+; LA64LARGE-NEXT:    add.d $a0, $t8, $a0
+; LA64LARGE-NEXT:    pcalau12i $ra, %pc_hi20(__tls_get_addr)
+; LA64LARGE-NEXT:    addi.d $t8, $zero, %pc_lo12(__tls_get_addr)
+; LA64LARGE-NEXT:    lu32i.d $t8, %pc64_lo20(__tls_get_addr)
+; LA64LARGE-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr)
+; LA64LARGE-NEXT:    add.d $ra, $t8, $ra
+; LA64LARGE-NEXT:    jirl $ra, $ra, 0
+; LA64LARGE-NEXT:    ld.w $zero, $a0, 0
+; LA64LARGE-NEXT:    addi.w $s0, $s0, 1
+; LA64LARGE-NEXT:    blt $s0, $fp, .LBB5_1
+; LA64LARGE-NEXT:  # %bb.2: # %ret
+; LA64LARGE-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64LARGE-NEXT:    addi.d $sp, $sp, 32
+; LA64LARGE-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %0 = load volatile i32, ptr @gd, align 4
+  %inc = add nuw nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %loop, label %ret
+
+ret:
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index 615cb5700ebf0733af854b5ce2cbda9b30d8395f..b0c77155c095b8928f5df782fe2c36fbaad4b076 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -119,6 +119,7 @@
 ; LAXX-NEXT:       Register Coalescer
 ; LAXX-NEXT:       Rename Disconnected Subregister Components
 ; LAXX-NEXT:       Machine Instruction Scheduler
+; LAXX-NEXT:       LoongArch Dead register definitions
 ; LAXX-NEXT:       Machine Block Frequency Analysis
 ; LAXX-NEXT:       Debug Variable Analysis
 ; LAXX-NEXT:       Live Stack Slot Analysis
diff --git a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
index 0effd469e3fb710f12406d07d6d51ff9f7eef6be..678546f1ef535a29a1a226639bf6b99ecd5112de 100644
--- a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
+++ b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
@@ -27,22 +27,22 @@ define void @foo() nounwind {
 ; MEDIUM_NO_SCH-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; MEDIUM_NO_SCH-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, %got_pc_lo12(G)
-; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, 0
+; MEDIUM_NO_SCH-NEXT:    ld.d $zero, $a0, 0
 ; MEDIUM_NO_SCH-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; MEDIUM_NO_SCH-NEXT:    addi.d $a0, $a0, %pc_lo12(g)
-; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, 0
+; MEDIUM_NO_SCH-NEXT:    ld.d $zero, $a0, 0
 ; MEDIUM_NO_SCH-NEXT:    ori $a0, $zero, 1
 ; MEDIUM_NO_SCH-NEXT:    pcaddu18i $ra, %call36(bar)
 ; MEDIUM_NO_SCH-NEXT:    jirl $ra, $ra, 0
 ; MEDIUM_NO_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(gd)
 ; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(gd)
-; MEDIUM_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_NO_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ld)
 ; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(ld)
-; MEDIUM_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_NO_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ie)
 ; MEDIUM_NO_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(ie)
-; MEDIUM_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_NO_SCH-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; MEDIUM_NO_SCH-NEXT:    addi.d $sp, $sp, 16
 ; MEDIUM_NO_SCH-NEXT:    ret
@@ -53,22 +53,22 @@ define void @foo() nounwind {
 ; MEDIUM_SCH-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; MEDIUM_SCH-NEXT:    pcalau12i $a0, %got_pc_hi20(G)
 ; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, %got_pc_lo12(G)
-; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, 0
+; MEDIUM_SCH-NEXT:    ld.d $zero, $a0, 0
 ; MEDIUM_SCH-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; MEDIUM_SCH-NEXT:    addi.d $a0, $a0, %pc_lo12(g)
-; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, 0
+; MEDIUM_SCH-NEXT:    ld.d $zero, $a0, 0
 ; MEDIUM_SCH-NEXT:    ori $a0, $zero, 1
 ; MEDIUM_SCH-NEXT:    pcaddu18i $ra, %call36(bar)
 ; MEDIUM_SCH-NEXT:    jirl $ra, $ra, 0
 ; MEDIUM_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(gd)
 ; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(gd)
-; MEDIUM_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ld)
 ; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(ld)
-; MEDIUM_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ie)
 ; MEDIUM_SCH-NEXT:    ld.d $a0, $a0, %ie_pc_lo12(ie)
-; MEDIUM_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; MEDIUM_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; MEDIUM_SCH-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; MEDIUM_SCH-NEXT:    addi.d $sp, $sp, 16
 ; MEDIUM_SCH-NEXT:    ret
@@ -82,13 +82,13 @@ define void @foo() nounwind {
 ; LARGE_NO_SCH-NEXT:    lu32i.d $t8, %got64_pc_lo20(G)
 ; LARGE_NO_SCH-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(G)
 ; LARGE_NO_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_NO_SCH-NEXT:    ld.d $a0, $a0, 0
+; LARGE_NO_SCH-NEXT:    ld.d $zero, $a0, 0
 ; LARGE_NO_SCH-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; LARGE_NO_SCH-NEXT:    addi.d $t8, $zero, %pc_lo12(g)
 ; LARGE_NO_SCH-NEXT:    lu32i.d $t8, %pc64_lo20(g)
 ; LARGE_NO_SCH-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(g)
 ; LARGE_NO_SCH-NEXT:    add.d $a0, $t8, $a0
-; LARGE_NO_SCH-NEXT:    ld.d $a0, $a0, 0
+; LARGE_NO_SCH-NEXT:    ld.d $zero, $a0, 0
 ; LARGE_NO_SCH-NEXT:    ori $a0, $zero, 1
 ; LARGE_NO_SCH-NEXT:    pcalau12i $ra, %got_pc_hi20(bar)
 ; LARGE_NO_SCH-NEXT:    addi.d $t8, $zero, %got_pc_lo12(bar)
@@ -101,19 +101,19 @@ define void @foo() nounwind {
 ; LARGE_NO_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(gd)
 ; LARGE_NO_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(gd)
 ; LARGE_NO_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_NO_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ld)
 ; LARGE_NO_SCH-NEXT:    addi.d $t8, $zero, %ie_pc_lo12(ld)
 ; LARGE_NO_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(ld)
 ; LARGE_NO_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(ld)
 ; LARGE_NO_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_NO_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ie)
 ; LARGE_NO_SCH-NEXT:    addi.d $t8, $zero, %ie_pc_lo12(ie)
 ; LARGE_NO_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(ie)
 ; LARGE_NO_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(ie)
 ; LARGE_NO_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_NO_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_NO_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_NO_SCH-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LARGE_NO_SCH-NEXT:    addi.d $sp, $sp, 16
 ; LARGE_NO_SCH-NEXT:    ret
@@ -127,13 +127,13 @@ define void @foo() nounwind {
 ; LARGE_SCH-NEXT:    lu32i.d $t8, %got64_pc_lo20(G)
 ; LARGE_SCH-NEXT:    lu52i.d $t8, $t8, %got64_pc_hi12(G)
 ; LARGE_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_SCH-NEXT:    ld.d $a0, $a0, 0
+; LARGE_SCH-NEXT:    ld.d $zero, $a0, 0
 ; LARGE_SCH-NEXT:    pcalau12i $a0, %pc_hi20(g)
 ; LARGE_SCH-NEXT:    addi.d $t8, $zero, %pc_lo12(g)
 ; LARGE_SCH-NEXT:    lu32i.d $t8, %pc64_lo20(g)
 ; LARGE_SCH-NEXT:    lu52i.d $t8, $t8, %pc64_hi12(g)
 ; LARGE_SCH-NEXT:    add.d $a0, $t8, $a0
-; LARGE_SCH-NEXT:    ld.d $a0, $a0, 0
+; LARGE_SCH-NEXT:    ld.d $zero, $a0, 0
 ; LARGE_SCH-NEXT:    ori $a0, $zero, 1
 ; LARGE_SCH-NEXT:    pcalau12i $ra, %got_pc_hi20(bar)
 ; LARGE_SCH-NEXT:    addi.d $t8, $zero, %got_pc_lo12(bar)
@@ -146,19 +146,19 @@ define void @foo() nounwind {
 ; LARGE_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(gd)
 ; LARGE_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(gd)
 ; LARGE_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ld)
 ; LARGE_SCH-NEXT:    addi.d $t8, $zero, %ie_pc_lo12(ld)
 ; LARGE_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(ld)
 ; LARGE_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(ld)
 ; LARGE_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_SCH-NEXT:    pcalau12i $a0, %ie_pc_hi20(ie)
 ; LARGE_SCH-NEXT:    addi.d $t8, $zero, %ie_pc_lo12(ie)
 ; LARGE_SCH-NEXT:    lu32i.d $t8, %ie64_pc_lo20(ie)
 ; LARGE_SCH-NEXT:    lu52i.d $t8, $t8, %ie64_pc_hi12(ie)
 ; LARGE_SCH-NEXT:    ldx.d $a0, $t8, $a0
-; LARGE_SCH-NEXT:    ldx.d $a0, $a0, $tp
+; LARGE_SCH-NEXT:    ldx.d $zero, $a0, $tp
 ; LARGE_SCH-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LARGE_SCH-NEXT:    addi.d $sp, $sp, 16
 ; LARGE_SCH-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/sextw-removal.ll b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
index 502b6cf88099e48f60cc7373e0a7d6c320a1c5e8..ba889fc8393b4af954531a1fa6b2e444fe13fa9c 100644
--- a/llvm/test/CodeGen/LoongArch/sextw-removal.ll
+++ b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
@@ -762,7 +762,6 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:  # %bb.1: # %.preheader
 ; CHECK-NEXT:    ori $a3, $zero, 1
 ; CHECK-NEXT:    addi.w $a2, $zero, -1
-; CHECK-NEXT:    lu32i.d $a2, 0
 ; CHECK-NEXT:    ori $a4, $zero, 1000
 ; CHECK-NEXT:    .p2align 4, , 16
 ; CHECK-NEXT:  .LBB13_2: # =>This Inner Loop Header: Depth=1
@@ -772,10 +771,9 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    addi.w $a3, $a3, 1
 ; CHECK-NEXT:    blt $a3, $a1, .LBB13_2
 ; CHECK-NEXT:  .LBB13_4:
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB13_5:
-; CHECK-NEXT:    addi.w $a0, $a2, 0
+; CHECK-NEXT:    move $a0, $a2
 ; CHECK-NEXT:    ret
 ;
 ; NORMV-LABEL: test14:
@@ -785,7 +783,6 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) {
 ; NORMV-NEXT:  # %bb.1: # %.preheader
 ; NORMV-NEXT:    ori $a3, $zero, 1
 ; NORMV-NEXT:    addi.w $a2, $zero, -1
-; NORMV-NEXT:    lu32i.d $a2, 0
 ; NORMV-NEXT:    ori $a4, $zero, 1000
 ; NORMV-NEXT:    .p2align 4, , 16
 ; NORMV-NEXT:  .LBB13_2: # =>This Inner Loop Header: Depth=1
@@ -795,13 +792,12 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) {
 ; NORMV-NEXT:    add.d $a0, $a3, $a0
 ; NORMV-NEXT:    addi.d $a3, $a3, 1
 ; NORMV-NEXT:    addi.w $a3, $a3, 0
-; NORMV-NEXT:    addi.d $a0, $a0, 0
+; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    blt $a3, $a1, .LBB13_2
 ; NORMV-NEXT:  .LBB13_4:
-; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    ret
 ; NORMV-NEXT:  .LBB13_5:
-; NORMV-NEXT:    addi.w $a0, $a2, 0
+; NORMV-NEXT:    move $a0, $a2
 ; NORMV-NEXT:    ret
   %3 = icmp sgt i32 %1, 1
   br i1 %3, label %4, label %12
@@ -830,8 +826,7 @@ define signext i32 @test14b(i32 %0, i32 signext %1) {
 ; CHECK-NEXT:    blt $a1, $a2, .LBB14_4
 ; CHECK-NEXT:  # %bb.1: # %.preheader
 ; CHECK-NEXT:    ori $a3, $zero, 1
-; CHECK-NEXT:    addi.w $a2, $zero, -1
-; CHECK-NEXT:    lu32i.d $a2, 0
+; CHECK-NEXT:    addi.d $a2, $zero, -1
 ; CHECK-NEXT:    ori $a4, $zero, 1000
 ; CHECK-NEXT:    .p2align 4, , 16
 ; CHECK-NEXT:  .LBB14_2: # =>This Inner Loop Header: Depth=1
@@ -854,8 +849,7 @@ define signext i32 @test14b(i32 %0, i32 signext %1) {
 ; NORMV-NEXT:    blt $a1, $a2, .LBB14_4
 ; NORMV-NEXT:  # %bb.1: # %.preheader
 ; NORMV-NEXT:    ori $a3, $zero, 1
-; NORMV-NEXT:    addi.w $a2, $zero, -1
-; NORMV-NEXT:    lu32i.d $a2, 0
+; NORMV-NEXT:    addi.d $a2, $zero, -1
 ; NORMV-NEXT:    ori $a4, $zero, 1000
 ; NORMV-NEXT:    .p2align 4, , 16
 ; NORMV-NEXT:  .LBB14_2: # =>This Inner Loop Header: Depth=1
@@ -900,8 +894,7 @@ define signext i32 @test14c(i32 zeroext %0, i32 signext %1) {
 ; CHECK-NEXT:    blt $a1, $a2, .LBB15_4
 ; CHECK-NEXT:  # %bb.1: # %.preheader
 ; CHECK-NEXT:    ori $a3, $zero, 1
-; CHECK-NEXT:    addi.w $a2, $zero, -1
-; CHECK-NEXT:    lu32i.d $a2, 0
+; CHECK-NEXT:    addi.d $a2, $zero, -1
 ; CHECK-NEXT:    ori $a4, $zero, 1000
 ; CHECK-NEXT:    .p2align 4, , 16
 ; CHECK-NEXT:  .LBB15_2: # =>This Inner Loop Header: Depth=1
@@ -924,8 +917,7 @@ define signext i32 @test14c(i32 zeroext %0, i32 signext %1) {
 ; NORMV-NEXT:    blt $a1, $a2, .LBB15_4
 ; NORMV-NEXT:  # %bb.1: # %.preheader
 ; NORMV-NEXT:    ori $a3, $zero, 1
-; NORMV-NEXT:    addi.w $a2, $zero, -1
-; NORMV-NEXT:    lu32i.d $a2, 0
+; NORMV-NEXT:    addi.d $a2, $zero, -1
 ; NORMV-NEXT:    ori $a4, $zero, 1000
 ; NORMV-NEXT:    .p2align 4, , 16
 ; NORMV-NEXT:  .LBB15_2: # =>This Inner Loop Header: Depth=1
@@ -971,7 +963,6 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) {
 ; CHECK-NEXT:  # %bb.1: # %.preheader
 ; CHECK-NEXT:    ori $a3, $zero, 1
 ; CHECK-NEXT:    addi.w $a2, $zero, -1
-; CHECK-NEXT:    lu32i.d $a2, 0
 ; CHECK-NEXT:    ori $a4, $zero, 1000
 ; CHECK-NEXT:    .p2align 4, , 16
 ; CHECK-NEXT:  .LBB16_2: # =>This Inner Loop Header: Depth=1
@@ -981,10 +972,9 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) {
 ; CHECK-NEXT:    addi.w $a3, $a3, 1
 ; CHECK-NEXT:    blt $a3, $a1, .LBB16_2
 ; CHECK-NEXT:  .LBB16_4:
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB16_5:
-; CHECK-NEXT:    addi.w $a0, $a2, 0
+; CHECK-NEXT:    move $a0, $a2
 ; CHECK-NEXT:    ret
 ;
 ; NORMV-LABEL: test14d:
@@ -994,7 +984,6 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) {
 ; NORMV-NEXT:  # %bb.1: # %.preheader
 ; NORMV-NEXT:    ori $a3, $zero, 1
 ; NORMV-NEXT:    addi.w $a2, $zero, -1
-; NORMV-NEXT:    lu32i.d $a2, 0
 ; NORMV-NEXT:    ori $a4, $zero, 1000
 ; NORMV-NEXT:    .p2align 4, , 16
 ; NORMV-NEXT:  .LBB16_2: # =>This Inner Loop Header: Depth=1
@@ -1004,13 +993,12 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) {
 ; NORMV-NEXT:    add.d $a0, $a3, $a0
 ; NORMV-NEXT:    addi.d $a3, $a3, 1
 ; NORMV-NEXT:    addi.w $a3, $a3, 0
-; NORMV-NEXT:    addi.d $a0, $a0, 0
+; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    blt $a3, $a1, .LBB16_2
 ; NORMV-NEXT:  .LBB16_4:
-; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    ret
 ; NORMV-NEXT:  .LBB16_5:
-; NORMV-NEXT:    addi.w $a0, $a2, 0
+; NORMV-NEXT:    move $a0, $a2
 ; NORMV-NEXT:    ret
   %zext = zext i31 %0 to i32
   %3 = icmp sgt i32 %1, 1
diff --git a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
index 18d17751a771962ed90d9b34cf86cab5b2ed991b..0a401ebe5f6b2aef1c2bb60d7d295e537848b425 100644
--- a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
+++ b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
@@ -742,12 +742,10 @@ define void @test_d8(ptr %P, ptr %S) nounwind {
 ; LA32F-NEXT:    st.w $a2, $sp, 32 # 4-byte Folded Spill
 ; LA32F-NEXT:    ld.w $a2, $a0, 52
 ; LA32F-NEXT:    st.w $a2, $sp, 28 # 4-byte Folded Spill
-; LA32F-NEXT:    ld.w $a2, $a0, 40
-; LA32F-NEXT:    st.w $a2, $sp, 16 # 4-byte Folded Spill
+; LA32F-NEXT:    ld.w $s8, $a0, 40
 ; LA32F-NEXT:    ld.w $a2, $a0, 44
-; LA32F-NEXT:    st.w $a2, $sp, 12 # 4-byte Folded Spill
-; LA32F-NEXT:    ld.w $a2, $a0, 32
-; LA32F-NEXT:    st.w $a2, $sp, 0 # 4-byte Folded Spill
+; LA32F-NEXT:    st.w $a2, $sp, 16 # 4-byte Folded Spill
+; LA32F-NEXT:    ld.w $s3, $a0, 32
 ; LA32F-NEXT:    ld.w $s4, $a0, 36
 ; LA32F-NEXT:    ld.w $s5, $a0, 24
 ; LA32F-NEXT:    ld.w $s6, $a0, 28
@@ -756,80 +754,77 @@ define void @test_d8(ptr %P, ptr %S) nounwind {
 ; LA32F-NEXT:    ld.w $s7, $a0, 8
 ; LA32F-NEXT:    ld.w $s0, $a0, 12
 ; LA32F-NEXT:    ld.w $a2, $a0, 0
-; LA32F-NEXT:    ld.w $a3, $a0, 4
+; LA32F-NEXT:    ld.w $a4, $a0, 4
 ; LA32F-NEXT:    move $fp, $a1
-; LA32F-NEXT:    lu12i.w $s8, 261888
+; LA32F-NEXT:    lu12i.w $a3, 261888
 ; LA32F-NEXT:    move $a0, $a2
-; LA32F-NEXT:    move $a1, $a3
+; LA32F-NEXT:    move $a1, $a4
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s8
 ; LA32F-NEXT:    bl %plt(__adddf3)
 ; LA32F-NEXT:    st.w $a0, $sp, 40 # 4-byte Folded Spill
 ; LA32F-NEXT:    st.w $a1, $sp, 36 # 4-byte Folded Spill
-; LA32F-NEXT:    lu12i.w $s3, 262144
+; LA32F-NEXT:    lu12i.w $a3, 262144
 ; LA32F-NEXT:    move $a0, $s7
 ; LA32F-NEXT:    move $a1, $s0
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s3
+; LA32F-NEXT:    move $s0, $a3
 ; LA32F-NEXT:    bl %plt(__adddf3)
 ; LA32F-NEXT:    st.w $a0, $sp, 24 # 4-byte Folded Spill
 ; LA32F-NEXT:    st.w $a1, $sp, 20 # 4-byte Folded Spill
-; LA32F-NEXT:    lu12i.w $s0, 262272
+; LA32F-NEXT:    lu12i.w $s7, 262272
 ; LA32F-NEXT:    move $a0, $s1
 ; LA32F-NEXT:    move $a1, $s2
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s0
+; LA32F-NEXT:    move $a3, $s7
 ; LA32F-NEXT:    bl %plt(__adddf3)
-; LA32F-NEXT:    st.w $a0, $sp, 8 # 4-byte Folded Spill
-; LA32F-NEXT:    st.w $a1, $sp, 4 # 4-byte Folded Spill
-; LA32F-NEXT:    lu12i.w $s7, 262400
+; LA32F-NEXT:    st.w $a0, $sp, 12 # 4-byte Folded Spill
+; LA32F-NEXT:    move $s2, $a1
+; LA32F-NEXT:    lu12i.w $a3, 262400
 ; LA32F-NEXT:    move $a0, $s5
 ; LA32F-NEXT:    move $a1, $s6
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s7
 ; LA32F-NEXT:    bl %plt(__adddf3)
 ; LA32F-NEXT:    move $s5, $a0
 ; LA32F-NEXT:    move $s6, $a1
-; LA32F-NEXT:    ld.w $a0, $sp, 0 # 4-byte Folded Reload
+; LA32F-NEXT:    move $a0, $s3
 ; LA32F-NEXT:    move $a1, $s4
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s8
+; LA32F-NEXT:    lu12i.w $a3, 261888
 ; LA32F-NEXT:    bl %plt(__adddf3)
-; LA32F-NEXT:    move $s4, $a0
-; LA32F-NEXT:    move $s8, $a1
-; LA32F-NEXT:    ld.w $a0, $sp, 16 # 4-byte Folded Reload
-; LA32F-NEXT:    ld.w $a1, $sp, 12 # 4-byte Folded Reload
+; LA32F-NEXT:    move $s3, $a0
+; LA32F-NEXT:    move $s4, $a1
+; LA32F-NEXT:    move $a0, $s8
+; LA32F-NEXT:    ld.w $a1, $sp, 16 # 4-byte Folded Reload
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s3
+; LA32F-NEXT:    move $a3, $s0
 ; LA32F-NEXT:    bl %plt(__adddf3)
-; LA32F-NEXT:    move $s3, $a0
-; LA32F-NEXT:    move $s1, $a1
+; LA32F-NEXT:    move $s8, $a0
+; LA32F-NEXT:    move $s0, $a1
 ; LA32F-NEXT:    ld.w $a0, $sp, 32 # 4-byte Folded Reload
 ; LA32F-NEXT:    ld.w $a1, $sp, 28 # 4-byte Folded Reload
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s0
+; LA32F-NEXT:    move $a3, $s7
 ; LA32F-NEXT:    bl %plt(__adddf3)
-; LA32F-NEXT:    move $s0, $a0
-; LA32F-NEXT:    move $s2, $a1
+; LA32F-NEXT:    move $s7, $a0
+; LA32F-NEXT:    move $s1, $a1
 ; LA32F-NEXT:    ld.w $a0, $sp, 48 # 4-byte Folded Reload
 ; LA32F-NEXT:    ld.w $a1, $sp, 44 # 4-byte Folded Reload
 ; LA32F-NEXT:    move $a2, $zero
-; LA32F-NEXT:    move $a3, $s7
+; LA32F-NEXT:    lu12i.w $a3, 262400
 ; LA32F-NEXT:    bl %plt(__adddf3)
 ; LA32F-NEXT:    st.w $a0, $fp, 56
 ; LA32F-NEXT:    st.w $a1, $fp, 60
-; LA32F-NEXT:    st.w $s0, $fp, 48
-; LA32F-NEXT:    st.w $s2, $fp, 52
-; LA32F-NEXT:    st.w $s3, $fp, 40
-; LA32F-NEXT:    st.w $s1, $fp, 44
-; LA32F-NEXT:    st.w $s4, $fp, 32
-; LA32F-NEXT:    st.w $s8, $fp, 36
+; LA32F-NEXT:    st.w $s7, $fp, 48
+; LA32F-NEXT:    st.w $s1, $fp, 52
+; LA32F-NEXT:    st.w $s8, $fp, 40
+; LA32F-NEXT:    st.w $s0, $fp, 44
+; LA32F-NEXT:    st.w $s3, $fp, 32
+; LA32F-NEXT:    st.w $s4, $fp, 36
 ; LA32F-NEXT:    st.w $s5, $fp, 24
 ; LA32F-NEXT:    st.w $s6, $fp, 28
-; LA32F-NEXT:    ld.w $a0, $sp, 8 # 4-byte Folded Reload
+; LA32F-NEXT:    ld.w $a0, $sp, 12 # 4-byte Folded Reload
 ; LA32F-NEXT:    st.w $a0, $fp, 16
-; LA32F-NEXT:    ld.w $a0, $sp, 4 # 4-byte Folded Reload
-; LA32F-NEXT:    st.w $a0, $fp, 20
+; LA32F-NEXT:    st.w $s2, $fp, 20
 ; LA32F-NEXT:    ld.w $a0, $sp, 24 # 4-byte Folded Reload
 ; LA32F-NEXT:    st.w $a0, $fp, 8
 ; LA32F-NEXT:    ld.w $a0, $sp, 20 # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/MIR/Generic/runPass.mir b/llvm/test/CodeGen/MIR/Generic/runPass.mir
index 54c1dd221bdb7748dea3e7f90fa0e8936967247e..75763c5389b09e4addf3ba33b9d005e98a29c3d7 100644
--- a/llvm/test/CodeGen/MIR/Generic/runPass.mir
+++ b/llvm/test/CodeGen/MIR/Generic/runPass.mir
@@ -1,6 +1,7 @@
 # RUN: llc -run-pass=greedy -debug-pass=Arguments -o - %s | FileCheck %s
 # RUN: llc -run-pass=regallocbasic -debug-pass=Arguments -o - %s | FileCheck %s
 # RUN: llc -run-pass=regallocfast -debug-pass=Arguments -o - %s | FileCheck %s
+# RUN: llc -passes=regallocfast -o - %s | FileCheck %s
 
 # Check that passes are initialized correctly, so that it's possible to
 # use -run-pass.
diff --git a/llvm/test/CodeGen/NVPTX/intr-range.ll b/llvm/test/CodeGen/NVPTX/intr-range.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2f3e08a039f52ef6fe2a1a41f88bf88a426e16c4
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/intr-range.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --version 5
+; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -mcpu=sm_20 -passes=nvvm-intr-range | FileCheck %s
+
+define i32 @test_maxntid() {
+; CHECK-LABEL: define i32 @test_maxntid(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call range(i32 0, 96) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+; CHECK-NEXT:    [[TMP3:%.*]] = call range(i32 0, 96) i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+; CHECK-NEXT:    [[TMP2:%.*]] = call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+; CHECK-NEXT:    [[TMP11:%.*]] = call range(i32 1, 97) i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+; CHECK-NEXT:    [[TMP4:%.*]] = call range(i32 1, 97) i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+; CHECK-NEXT:    [[TMP6:%.*]] = call range(i32 1, 65) i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP10]], [[TMP6]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %2 = call i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+  %3 = call i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+  %4 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+  %5 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+  %6 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+  %7 = add i32 %1, %2
+  %8 = add i32 %7, %3
+  %9 = add i32 %8, %4
+  %10 = add i32 %9, %5
+  %11 = add i32 %10, %6
+  ret i32 %11
+}
+
+define i32 @test_reqntid() {
+; CHECK-LABEL: define i32 @test_reqntid(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+; CHECK-NEXT:    [[TMP5:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+; CHECK-NEXT:    [[TMP2:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+; CHECK-NEXT:    [[TMP4:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+; CHECK-NEXT:    [[TMP3:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+; CHECK-NEXT:    [[TMP6:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP8]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], [[TMP6]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %2 = call i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+  %3 = call i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+  %4 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+  %5 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+  %6 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+  %7 = add i32 %1, %2
+  %8 = add i32 %7, %3
+  %9 = add i32 %8, %4
+  %10 = add i32 %9, %5
+  %11 = add i32 %10, %6
+  ret i32 %5
+}
+
+;; A case like this could occur if a function with the sreg intrinsic was
+;; inlined into a kernel where the tid metadata is present, ensure the range is
+;; updated.
+define i32 @test_inlined() {
+; CHECK-LABEL: define i32 @test_inlined(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call range(i32 0, 4) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  ret i32 %1
+}
+
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+
+declare i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+declare i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+declare i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+
+!nvvm.annotations = !{!0, !1, !2}
+!0 = !{ptr @test_maxntid, !"kernel", i32 1, !"maxntidx", i32 32, !"maxntidz", i32 3}
+!1 = !{ptr @test_reqntid, !"kernel", i32 1, !"reqntidx", i32 20}
+!2 = !{ptr @test_inlined, !"kernel", i32 1, !"maxntidx", i32 4}
diff --git a/llvm/test/CodeGen/NVPTX/intrinsic-old.ll b/llvm/test/CodeGen/NVPTX/intrinsic-old.ll
index 3930e6d7741838c8190e92da8365ec7adc2844ab..85f7817f08dc49a80d380fd905ece7da057e20e0 100644
--- a/llvm/test/CodeGen/NVPTX/intrinsic-old.ll
+++ b/llvm/test/CodeGen/NVPTX/intrinsic-old.ll
@@ -1,21 +1,13 @@
 ; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck -allow-deprecated-dag-overlap %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck -allow-deprecated-dag-overlap %s
 ; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -passes=nvvm-intr-range \
-; RUN:   | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_20 %s
-; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -passes=nvvm-intr-range \
-; RUN:   | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_20 %s
-; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda \
-; RUN:    -passes=nvvm-intr-range -nvvm-intr-range-sm=30 \
-; RUN:   | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_30 %s
-; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda \
-; RUN:    -passes=nvvm-intr-range -nvvm-intr-range-sm=30 \
-; RUN:   | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_30 %s
+; RUN:   | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE %s
 ; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -march=nvptx -mcpu=sm_20 | %ptxas-verify %}
 ; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
 define ptx_device i32 @test_tid_x() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.x;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range ![[BLK_IDX_XY:[0-9]+]]
+; RANGE: call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 	ret i32 %x
@@ -23,7 +15,7 @@ define ptx_device i32 @test_tid_x() {
 
 define ptx_device i32 @test_tid_y() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.y;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.y(), !range ![[BLK_IDX_XY]]
+; RANGE: call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.y()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.tid.y()
 	ret i32 %x
@@ -31,7 +23,7 @@ define ptx_device i32 @test_tid_y() {
 
 define ptx_device i32 @test_tid_z() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.z;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.z(), !range ![[BLK_IDX_Z:[0-9]+]]
+; RANGE: call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.z()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.tid.z()
 	ret i32 %x
@@ -46,7 +38,7 @@ define ptx_device i32 @test_tid_w() {
 
 define ptx_device i32 @test_ntid_x() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.x;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range ![[BLK_SIZE_XY:[0-9]+]]
+; RANGE: call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
 	ret i32 %x
@@ -54,7 +46,7 @@ define ptx_device i32 @test_ntid_x() {
 
 define ptx_device i32 @test_ntid_y() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.y;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.y(), !range ![[BLK_SIZE_XY]]
+; RANGE: call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
 	ret i32 %x
@@ -62,7 +54,7 @@ define ptx_device i32 @test_ntid_y() {
 
 define ptx_device i32 @test_ntid_z() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.z;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.z(), !range ![[BLK_SIZE_Z:[0-9]+]]
+; RANGE: call range(i32 1, 65) i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
 	ret i32 %x
@@ -77,7 +69,7 @@ define ptx_device i32 @test_ntid_w() {
 
 define ptx_device i32 @test_laneid() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %laneid;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.laneid(), !range ![[LANEID:[0-9]+]]
+; RANGE: call range(i32 0, 32) i32 @llvm.nvvm.read.ptx.sreg.laneid()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.laneid()
 	ret i32 %x
@@ -85,7 +77,7 @@ define ptx_device i32 @test_laneid() {
 
 define ptx_device i32 @test_warpsize() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, WARP_SZ;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.warpsize(), !range ![[WARPSIZE:[0-9]+]]
+; RANGE: call range(i32 32, 33) i32 @llvm.nvvm.read.ptx.sreg.warpsize()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.warpsize()
 	ret i32 %x
@@ -107,7 +99,7 @@ define ptx_device i32 @test_nwarpid() {
 
 define ptx_device i32 @test_ctaid_y() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.y;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y(), !range ![[GRID_IDX_YZ:[0-9]+]]
+; RANGE: call range(i32 0, 65535) i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
 	ret i32 %x
@@ -115,7 +107,7 @@ define ptx_device i32 @test_ctaid_y() {
 
 define ptx_device i32 @test_ctaid_z() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.z;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z(), !range ![[GRID_IDX_YZ]]
+; RANGE: call range(i32 0, 65535) i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
 	ret i32 %x
@@ -123,8 +115,7 @@ define ptx_device i32 @test_ctaid_z() {
 
 define ptx_device i32 @test_ctaid_x() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.x;
-; RANGE_30: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range ![[GRID_IDX_X:[0-9]+]]
-; RANGE_20: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range ![[GRID_IDX_YZ]]
+; RANGE: call range(i32 0, 2147483647) i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
 	ret i32 %x
@@ -139,7 +130,7 @@ define ptx_device i32 @test_ctaid_w() {
 
 define ptx_device i32 @test_nctaid_y() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.y;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y(), !range ![[GRID_SIZE_YZ:[0-9]+]]
+; RANGE: call range(i32 1, 65536) i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
 	ret i32 %x
@@ -147,7 +138,7 @@ define ptx_device i32 @test_nctaid_y() {
 
 define ptx_device i32 @test_nctaid_z() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.z;
-; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z(), !range ![[GRID_SIZE_YZ]]
+; RANGE: call range(i32 1, 65536) i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
 	ret i32 %x
@@ -155,8 +146,7 @@ define ptx_device i32 @test_nctaid_z() {
 
 define ptx_device i32 @test_nctaid_x() {
 ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.x;
-; RANGE_30: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x(), !range ![[GRID_SIZE_X:[0-9]+]]
-; RANGE_20: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x(), !range ![[GRID_SIZE_YZ]]
+; RANGE: call range(i32 1, -2147483648) i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
 ; CHECK: ret;
 	%x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
 	ret i32 %x
@@ -327,14 +317,3 @@ declare void @llvm.nvvm.bar.sync(i32 %i)
 
 !0 = !{i32 0, i32 19}
 ; RANGE-DAG: ![[ALREADY]] = !{i32 0, i32 19}
-; RANGE-DAG: ![[BLK_IDX_XY]] = !{i32 0, i32 1024}
-; RANGE-DAG: ![[BLK_IDX_XY]] = !{i32 0, i32 1024}
-; RANGE-DAG: ![[BLK_IDX_Z]] = !{i32 0, i32 64}
-; RANGE-DAG: ![[BLK_SIZE_XY]] = !{i32 1, i32 1025}
-; RANGE-DAG: ![[BLK_SIZE_Z]] = !{i32 1, i32 65}
-; RANGE-DAG: ![[LANEID]] = !{i32 0, i32 32}
-; RANGE-DAG: ![[WARPSIZE]] = !{i32 32, i32 33}
-; RANGE_30-DAG: ![[GRID_IDX_X]] = !{i32 0, i32 2147483647}
-; RANGE-DAG: ![[GRID_IDX_YZ]] = !{i32 0, i32 65535}
-; RANGE_30-DAG: ![[GRID_SIZE_X]] = !{i32 1, i32 -2147483648}
-; RANGE-DAG: ![[GRID_SIZE_YZ]] = !{i32 1, i32 65536}
diff --git a/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll b/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll
index e4deef157be81c2eeb6640c075fd111177704d2a..87565df1506b9475baffca5b02926f20db962d41 100644
--- a/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll
+++ b/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll
@@ -7,7 +7,9 @@ declare ptr @llvm.frameaddress(i32) nounwind readnone
 define ptr @g2() nounwind readnone {
 ; CHECK-LABEL: g2:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwu 1, -16(1)
 ; CHECK-NEXT:    lwz 3, 0(1)
+; CHECK-NEXT:    addi 1, 1, 16
 ; CHECK-NEXT:    blr
 entry:
   %0 = tail call ptr @llvm.frameaddress(i32 1)    ; <ptr> [#uses=1]
diff --git a/llvm/test/CodeGen/PowerPC/frameaddr.ll b/llvm/test/CodeGen/PowerPC/frameaddr.ll
index 74263393442c8e10402fa15189e9aaa2dc7d7a54..180f736c5ddbddf3911cb9ede01fe452734545ca 100644
--- a/llvm/test/CodeGen/PowerPC/frameaddr.ll
+++ b/llvm/test/CodeGen/PowerPC/frameaddr.ll
@@ -1,38 +1,235 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare void @llvm.eh.sjlj.longjmp(ptr) #1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=powerpc-ibm-aix-xcoff -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=AIX32
+; RUN: llc -mtriple=powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=AIX64
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=LE
 
 define ptr @main() #0 {
+; AIX32-LABEL: main:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    mr 3, 1
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: main:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    mr 3, 1
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: main:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    stdu 1, -32(1)
+; LE-NEXT:    mr 3, 1
+; LE-NEXT:    addi 1, 1, 32
+; LE-NEXT:    blr
 entry:
   %0 = call ptr @llvm.frameaddress(i32 0)
   ret ptr %0
+}
 
-; CHECK: @main
-; CHECK: mr 3, 1
+define ptr @foo_naked() #3 { ; naked
+; AIX32-LABEL: foo_naked:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    lwz 3, 0(1)
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: foo_naked:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    ld 3, 0(1)
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: foo_naked:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    ld 3, 0(1)
+; LE-NEXT:    blr
+entry:
+  %0 = call ptr @llvm.frameaddress(i32 1)
+  ret ptr %0
 }
 
-define ptr @foo() #3 { ; naked
+define ptr @foo1() #0 {
+; AIX32-LABEL: foo1:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 3, 0(1)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: foo1:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 3, 0(1)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: foo1:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    stdu 1, -32(1)
+; LE-NEXT:    ld 3, 0(1)
+; LE-NEXT:    addi 1, 1, 32
+; LE-NEXT:    blr
 entry:
-  %0 = call ptr @llvm.frameaddress(i32 0)
+  %0 = call ptr @llvm.frameaddress(i32 1)
   ret ptr %0
+}
 
-; CHECK: @foo
-; CHECK: mr 3, 1
+define ptr @foo2() #0 {
+; AIX32-LABEL: foo2:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 3, 0(1)
+; AIX32-NEXT:    lwz 3, 0(3)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: foo2:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 3, 0(1)
+; AIX64-NEXT:    ld 3, 0(3)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: foo2:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    stdu 1, -32(1)
+; LE-NEXT:    ld 3, 0(1)
+; LE-NEXT:    ld 3, 0(3)
+; LE-NEXT:    addi 1, 1, 32
+; LE-NEXT:    blr
+entry:
+  %0 = call ptr @llvm.frameaddress(i32 2)
+  ret ptr %0
 }
 
-define ptr @bar() #0 {
+define ptr @bar0() #0 {
+; AIX32-LABEL: bar0:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stw 31, -4(1)
+; AIX32-NEXT:    stw 0, 8(1)
+; AIX32-NEXT:    lis 0, -2
+; AIX32-NEXT:    ori 0, 0, 31008
+; AIX32-NEXT:    stwux 1, 1, 0
+; AIX32-NEXT:    mr 31, 1
+; AIX32-NEXT:    addi 3, 31, 60
+; AIX32-NEXT:    bl .use[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    mr 3, 31
+; AIX32-NEXT:    lwz 1, 0(1)
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    lwz 31, -4(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: bar0:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    std 31, -8(1)
+; AIX64-NEXT:    std 0, 16(1)
+; AIX64-NEXT:    lis 0, -2
+; AIX64-NEXT:    ori 0, 0, 30944
+; AIX64-NEXT:    stdux 1, 1, 0
+; AIX64-NEXT:    mr 31, 1
+; AIX64-NEXT:    addi 3, 31, 120
+; AIX64-NEXT:    bl .use[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    mr 3, 31
+; AIX64-NEXT:    ld 1, 0(1)
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    ld 31, -8(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: bar0:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    mflr 0
+; LE-NEXT:    std 31, -8(1)
+; LE-NEXT:    std 0, 16(1)
+; LE-NEXT:    lis 0, -2
+; LE-NEXT:    ori 0, 0, 31024
+; LE-NEXT:    stdux 1, 1, 0
+; LE-NEXT:    mr 31, 1
+; LE-NEXT:    addi 3, 31, 40
+; LE-NEXT:    bl use
+; LE-NEXT:    nop
+; LE-NEXT:    mr 3, 31
+; LE-NEXT:    ld 1, 0(1)
+; LE-NEXT:    ld 0, 16(1)
+; LE-NEXT:    ld 31, -8(1)
+; LE-NEXT:    mtlr 0
+; LE-NEXT:    blr
 entry:
   %x = alloca [100000 x i8]                       ; <ptr> [#uses=1]
   call void @use(ptr %x) nounwind
   %0 = call ptr @llvm.frameaddress(i32 0)
   ret ptr %0
+}
 
-; Note that if we start eliminating non-leaf frame pointers by default, this
-; will need to be updated.
-; CHECK: @bar
-; CHECK: mr 3, 31
+define ptr @bar1() #0 {
+; AIX32-LABEL: bar1:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stw 31, -4(1)
+; AIX32-NEXT:    stw 0, 8(1)
+; AIX32-NEXT:    lis 0, -2
+; AIX32-NEXT:    ori 0, 0, 31008
+; AIX32-NEXT:    stwux 1, 1, 0
+; AIX32-NEXT:    mr 31, 1
+; AIX32-NEXT:    addi 3, 31, 60
+; AIX32-NEXT:    bl .use[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 0(31)
+; AIX32-NEXT:    lwz 1, 0(1)
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    lwz 31, -4(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; AIX64-LABEL: bar1:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    std 31, -8(1)
+; AIX64-NEXT:    std 0, 16(1)
+; AIX64-NEXT:    lis 0, -2
+; AIX64-NEXT:    ori 0, 0, 30944
+; AIX64-NEXT:    stdux 1, 1, 0
+; AIX64-NEXT:    mr 31, 1
+; AIX64-NEXT:    addi 3, 31, 120
+; AIX64-NEXT:    bl .use[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    ld 3, 0(31)
+; AIX64-NEXT:    ld 1, 0(1)
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    ld 31, -8(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; LE-LABEL: bar1:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    mflr 0
+; LE-NEXT:    std 31, -8(1)
+; LE-NEXT:    std 0, 16(1)
+; LE-NEXT:    lis 0, -2
+; LE-NEXT:    ori 0, 0, 31024
+; LE-NEXT:    stdux 1, 1, 0
+; LE-NEXT:    mr 31, 1
+; LE-NEXT:    addi 3, 31, 40
+; LE-NEXT:    bl use
+; LE-NEXT:    nop
+; LE-NEXT:    ld 3, 0(31)
+; LE-NEXT:    ld 1, 0(1)
+; LE-NEXT:    ld 0, 16(1)
+; LE-NEXT:    ld 31, -8(1)
+; LE-NEXT:    mtlr 0
+; LE-NEXT:    blr
+entry:
+  %x = alloca [100000 x i8]                       ; <ptr> [#uses=1]
+  call void @use(ptr %x) nounwind
+  %0 = call ptr @llvm.frameaddress(i32 1)
+  ret ptr %0
 }
 
 declare void @use(ptr)
@@ -43,4 +240,3 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-lea
 attributes #1 = { noreturn nounwind }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind naked "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-
diff --git a/llvm/test/CodeGen/PowerPC/spill-nor0.mir b/llvm/test/CodeGen/PowerPC/spill-nor0.mir
index 2f50ff3701d133643ba251e660fb0a5af984c37b..6658ca156d4d299f2c02dffee3c0dbd30e602315 100644
--- a/llvm/test/CodeGen/PowerPC/spill-nor0.mir
+++ b/llvm/test/CodeGen/PowerPC/spill-nor0.mir
@@ -1,4 +1,5 @@
 # RUN: llc -o - %s -mtriple=powerpc64-- -run-pass=regallocfast | FileCheck %s
+# RUN: llc -o - %s -mtriple=powerpc64-- -passes=regallocfast | FileCheck %s
 ---
 # CHECK-LABEL: name: func
 name: func
diff --git a/llvm/test/CodeGen/PowerPC/toc-data-common.ll b/llvm/test/CodeGen/PowerPC/toc-data-common.ll
index 7747f2eecc935ee48a290701e4eeb3dd641d9efe..3b7ca44f5bbb031450360209c10c185148a5f51c 100644
--- a/llvm/test/CodeGen/PowerPC/toc-data-common.ll
+++ b/llvm/test/CodeGen/PowerPC/toc-data-common.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s -DINSTR=lwz --check-prefix=CHECK
-; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s -DINSTR=ld --check-prefix=CHECK
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-64
 
 ; RUN: llc -filetype=obj -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s -o %t32.o
 ; RUN: llvm-objdump -t --symbol-description %t32.o | FileCheck %s --check-prefix=OBJ32
@@ -15,16 +15,28 @@
 
 define void @set(i32 noundef %_a) {
 ; CHECK-LABEL: set:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: la 4, a2[TD](2)
-; CHECK-NEXT: la 5, a1[TD](2)
-; CHECK-NEXT: stw 3, 0(4)
-; CHECK-NEXT: [[INSTR]] 4, L..C0(2) # @a4
-; CHECK-NEXT: stw 3, 0(5)
-; CHECK-NEXT: [[INSTR]] 5, L..C1(2) # @a3
-; CHECK-NEXT: stw 3, 0(4)
-; CHECK-NEXT: stw 3, 0(5)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    la 4, a2[TD](2)
+; CHECK-NEXT:    lwz 5, L..C0(2) # @a4
+; CHECK-NEXT:    stw 3, 0(4)
+; CHECK-NEXT:    la 4, a1[TD](2)
+; CHECK-NEXT:    stw 3, 0(4)
+; CHECK-NEXT:    lwz 4, L..C1(2) # @a3
+; CHECK-NEXT:    stw 3, 0(5)
+; CHECK-NEXT:    stw 3, 0(4)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: set:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    la 4, a2[TD](2)
+; CHECK-64-NEXT:    ld 5, L..C0(2) # @a4
+; CHECK-64-NEXT:    stw 3, 0(4)
+; CHECK-64-NEXT:    la 4, a1[TD](2)
+; CHECK-64-NEXT:    stw 3, 0(4)
+; CHECK-64-NEXT:    ld 4, L..C1(2) # @a3
+; CHECK-64-NEXT:    stw 3, 0(5)
+; CHECK-64-NEXT:    stw 3, 0(4)
+; CHECK-64-NEXT:    blr
 entry:
 store i32 %_a, ptr @a2, align 4
 store i32 %_a, ptr @a1, align 4
@@ -35,10 +47,16 @@ ret void
 
 define i32 @get1() {
 ; CHECK-LABEL: get1:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: la 3, a2[TD](2)
-; CHECK-NEXT: lwz 3, 0(3)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    la 3, a2[TD](2)
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: get1:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    la 3, a2[TD](2)
+; CHECK-64-NEXT:    lwz 3, 0(3)
+; CHECK-64-NEXT:    blr
 entry:
 %0 = load i32, ptr @a2, align 4
 ret i32 %0
@@ -46,10 +64,16 @@ ret i32 %0
 
 define i32 @get2() {
 ; CHECK-LABEL: get2:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: la 3, a1[TD](2)
-; CHECK-NEXT: lwz 3, 0(3)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    la 3, a1[TD](2)
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: get2:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    la 3, a1[TD](2)
+; CHECK-64-NEXT:    lwz 3, 0(3)
+; CHECK-64-NEXT:    blr
 entry:
 %0 = load i32, ptr @a1, align 4
 ret i32 %0
@@ -57,10 +81,16 @@ ret i32 %0
 
 define i32 @get3() {
 ; CHECK-LABEL: get3:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: [[INSTR]] 3, L..C0(2) # @a4
-; CHECK-NEXT: lwz 3, 0(3)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 3, L..C0(2) # @a4
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: get3:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    ld 3, L..C0(2) # @a4
+; CHECK-64-NEXT:    lwz 3, 0(3)
+; CHECK-64-NEXT:    blr
 entry:
 %0 = load i32, ptr @a4, align 4
 ret i32 %0
@@ -68,10 +98,16 @@ ret i32 %0
 
 define i32 @get4() {
 ; CHECK-LABEL: get4:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: [[INSTR]] 3, L..C1(2) # @a3
-; CHECK-NEXT: lwz 3, 0(3)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 3, L..C1(2) # @a3
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: get4:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    ld 3, L..C1(2) # @a3
+; CHECK-64-NEXT:    lwz 3, 0(3)
+; CHECK-64-NEXT:    blr
 entry:
 %0 = load i32, ptr @a3, align 4
 ret i32 %0
@@ -79,36 +115,56 @@ ret i32 %0
 
 define nonnull ptr @escape1() {
 ; CHECK-LABEL: escape1:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: la 3, a2[TD](2)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    la 3, a2[TD](2)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: escape1:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    la 3, a2[TD](2)
+; CHECK-64-NEXT:    blr
 entry:
 ret ptr @a2
 }
 
 define nonnull ptr @escape2() {
 ; CHECK-LABEL: escape2:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: la 3, a1[TD](2)
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    la 3, a1[TD](2)
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: escape2:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    la 3, a1[TD](2)
+; CHECK-64-NEXT:    blr
 entry:
 ret ptr @a1
 }
 
 define nonnull ptr @escape3() {
 ; CHECK-LABEL: escape3:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: [[INSTR]] 3, L..C0(2) # @a4
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 3, L..C0(2) # @a4
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: escape3:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    ld 3, L..C0(2) # @a4
+; CHECK-64-NEXT:    blr
 entry:
 ret ptr @a4
 }
 
 define nonnull ptr @escape4() {
 ; CHECK-LABEL: escape4:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: [[INSTR]] 3, L..C1(2) # @a3
-; CHECK-NEXT: blr
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 3, L..C1(2) # @a3
+; CHECK-NEXT:    blr
+;
+; CHECK-64-LABEL: escape4:
+; CHECK-64:       # %bb.0: # %entry
+; CHECK-64-NEXT:    ld 3, L..C1(2) # @a3
+; CHECK-64-NEXT:    blr
 entry:
 ret ptr @a3
 }
diff --git a/llvm/test/CodeGen/PowerPC/toc-data.ll b/llvm/test/CodeGen/PowerPC/toc-data.ll
index 12286657488dbad908e9aa7cf6cd0d4633ab0107..ee1dde190bb26a03ed7cd1b31b0a50c187ee33fc 100644
--- a/llvm/test/CodeGen/PowerPC/toc-data.ll
+++ b/llvm/test/CodeGen/PowerPC/toc-data.ll
@@ -36,7 +36,7 @@ define dso_local void @write_int(i32 signext %in) {
     ret void
 }
 ; CHECK32: name:            write_int
-; CHECK32:      %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc @i, $r2
+; CHECK32:      %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc $r2, @i
 ; CHECK32-NEXT: STW %{{[0-9]+}}, 0, killed %[[SCRATCH]] :: (store (s32) into @i)
 
 ; TEST32:         .write_int:
@@ -44,12 +44,12 @@ define dso_local void @write_int(i32 signext %in) {
 ; TEST32-NEXT:      stw 3, 0(4)
 
 ; CHECK64: name:            write_int
-; CHECK64:      %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2
+; CHECK64:      %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i
 ; CHECK64-NEXT: STW8 %{{[0-9]+}}, 0, killed %[[SCRATCH]] :: (store (s32) into @i)
 
 ; CHECK64-NOOPT:  name: write_int
 ; CHECK64-NOOPT:    %[[SUBREG:[0-9]+]]:gprc = COPY %{{[0-9]}}.sub_32
-; CHECK64-NOOPT:    %[[ADDR:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2
+; CHECK64-NOOPT:    %[[ADDR:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i
 ; CHECK64-NOOPT:    STW %[[SUBREG]], 0, %[[ADDR]]
 
 ; TEST64:         .write_int:
@@ -128,7 +128,7 @@ define dso_local float @read_float() {
     ret float %0
 }
 ; CHECK32: name:            read_float
-; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc @f, $r2
+; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc $r2, @f
 ; CHECK32: %{{[0-9]+}}:f4rc = LFS 0, killed %[[SCRATCH]] :: (dereferenceable load (s32) from @f)
 
 ; TEST32:       .read_float:
@@ -136,11 +136,11 @@ define dso_local float @read_float() {
 ; TEST32-NEXT:    lfs 1, 0(3)
 
 ; CHECK64: name:            read_float
-; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @f, $x2
+; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @f
 ; CHECK64: %{{[0-9]+}}:f4rc = LFS 0, killed %[[SCRATCH]] :: (dereferenceable load (s32) from @f)
 
 ; CHECK64-NOOPT: name:            read_float
-; CHECK64-NOOPT:   %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @f, $x2
+; CHECK64-NOOPT:   %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @f
 ; CHECK64-NOOPT:   %{{[0-9]+}}:f4rc = LFS 0, %[[SCRATCH]]
 
 ; TEST64:       .read_float:
@@ -217,18 +217,18 @@ define dso_local nonnull ptr @addr() {
     ret ptr @i
 }
 ; CHECK32: name:            addr
-; CHECK32:       %[[SCRATCH:[0-9]+]]:gprc = ADDItoc @i, $r2
+; CHECK32:       %[[SCRATCH:[0-9]+]]:gprc = ADDItoc $r2, @i
 ; CHECK32-NEXT:  $r3 = COPY %[[SCRATCH]]
 
 ; TEST32:       .addr
 ; TEST32:         la 3, i[TD](2)
 
 ; CHECK64: name:            addr
-; CHECK64:       %[[SCRATCH:[0-9]+]]:g8rc = ADDItoc8 @i, $x2
+; CHECK64:       %[[SCRATCH:[0-9]+]]:g8rc = ADDItoc8 $x2, @i
 ; CHECK64-NEXT:  $x3 = COPY %[[SCRATCH]]
 
 ; CHECK64-NOOPT: name:            addr
-; CHECK64-NOOPT:   %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2
+; CHECK64-NOOPT:   %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i
 ; CHECK64-NOOPT:   $x3 = COPY %[[SCRATCH]]
 
 ; TEST64:       .addr
diff --git a/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll b/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c982713d4f8d30230efa5dfc8e9a8f5fd3a48102
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -passes='default<O3>' < %s | FileCheck %s
+
+target datalayout = "E-m:a-p:32:32-Fi32-i64:64-n32"
+target triple = "powerpc-ibm-aix7.2.0.0"
+
+%struct.widget = type { i8, i8, i8 }
+
+; CHECK: @global = {{.*}}constant %struct.widget { i8 4, i8 0, i8 0 }, align 8 #0
+@global = constant %struct.widget { i8 4, i8 0, i8 0 }, align 4 #0
+
+define void @baz() #1 {
+bb:
+  call void @snork(ptr @global)
+  ret void
+}
+
+define void @snork(ptr byval(%struct.widget) %arg) #1 {
+bb:
+  %load = load volatile ptr, ptr null, align 4
+  ret void
+}
+
+attributes #0 = { "toc-data" }
+attributes #1 = { "target-cpu"="pwr7" "target-features"="+altivec,+bpermd,+extdiv,+isa-v206-instructions,+vsx,-aix-shared-lib-tls-model-opt,-aix-small-local-dynamic-tls,-aix-small-local-exec-tls,-crbits,-crypto,-direct-move,-htm,-isa-v207-instructions,-isa-v30-instructions,-power8-vector,-power9-vector,-privileged,-quadword-atomics,-rop-protect,-spe" }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0a0828e51893f85cb2911fa49958fffc17482da4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+f -global-isel -stop-after=irtranslator < %s \
+; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfh -global-isel -stop-after=irtranslator < %s \
+; RUN:   | FileCheck -check-prefix=RV32IZFH %s
+
+define half @callee_half_in_regs(half %x) nounwind {
+  ; RV32I-LABEL: name: callee_half_in_regs
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: callee_half_in_regs
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: callee_half_in_regs
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  ret half %x
+}
+
+define half @caller_half_in_regs(half %x) nounwind {
+  ; RV32I-LABEL: name: caller_half_in_regs
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @caller_half_in_regs, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+  ; RV32I-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT1]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: caller_half_in_regs
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoCALL target-flags(riscv-call) @caller_half_in_regs, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_f, implicit-def $f10_f
+  ; RV32IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT1]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: caller_half_in_regs
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   PseudoCALL target-flags(riscv-call) @caller_half_in_regs, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_h, implicit-def $f10_h
+  ; RV32IZFH-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  %y = call half @caller_half_in_regs(half %x)
+  ret half %y
+}
+
+define half @callee_half_mixed_with_int(i32 %x0, half %x) nounwind {
+  ; RV32I-LABEL: name: callee_half_mixed_with_int
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: callee_half_mixed_with_int
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $x10, $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: callee_half_mixed_with_int
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $x10, $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  ret half %x
+}
+
+define half @caller_half_mixed_with_int(half %x, i32 %x0) nounwind {
+  ; RV32I-LABEL: name: caller_half_mixed_with_int
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_mixed_with_int, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; RV32I-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT1]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: caller_half_mixed_with_int
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $x10, $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_mixed_with_int, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $f10_f, implicit-def $f10_f
+  ; RV32IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT1]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: caller_half_mixed_with_int
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $x10, $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IZFH-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_mixed_with_int, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $f10_h, implicit-def $f10_h
+  ; RV32IZFH-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   [[COPY2:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY2]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  %y = call half @callee_half_mixed_with_int(i32 %x0, half %x)
+  ret half %y
+}
+
+define half @callee_half_return_stack1(i32 %v1, i32 %v2, i32 %v3, i32 %v4, i32 %v5, i32 %v6, i32 %v7, i32 %v8, half %x) nounwind {
+  ; RV32I-LABEL: name: callee_half_return_stack1
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32I-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32I-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32I-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32I-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $x17
+  ; RV32I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s32) from %fixed-stack.0, align 16)
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: callee_half_return_stack1
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32IF-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32IF-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32IF-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32IF-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32IF-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $x17
+  ; RV32IF-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: callee_half_return_stack1
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32IZFH-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32IZFH-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32IZFH-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32IZFH-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32IZFH-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32IZFH-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $x17
+  ; RV32IZFH-NEXT:   [[COPY8:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY8]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  ret half %x
+}
+
+define half @caller_half_return_stack1(i32 %v1, half %x) nounwind {
+  ; RV32I-LABEL: name: caller_half_return_stack1
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; RV32I-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; RV32I-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; RV32I-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; RV32I-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV32I-NEXT:   ADJCALLSTACKDOWN 4, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV32I-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C7]](s32)
+  ; RV32I-NEXT:   G_STORE [[ANYEXT]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; RV32I-NEXT:   $x10 = COPY [[C]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[C1]](s32)
+  ; RV32I-NEXT:   $x12 = COPY [[C2]](s32)
+  ; RV32I-NEXT:   $x13 = COPY [[COPY]](s32)
+  ; RV32I-NEXT:   $x14 = COPY [[C3]](s32)
+  ; RV32I-NEXT:   $x15 = COPY [[C4]](s32)
+  ; RV32I-NEXT:   $x16 = COPY [[C5]](s32)
+  ; RV32I-NEXT:   $x17 = COPY [[C6]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack1, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; RV32I-NEXT:   ADJCALLSTACKUP 4, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
+  ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT1]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: caller_half_return_stack1
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $x10, $f10_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32IF-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32IF-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32IF-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; RV32IF-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; RV32IF-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; RV32IF-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; RV32IF-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV32IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $x10 = COPY [[C]](s32)
+  ; RV32IF-NEXT:   $x11 = COPY [[C1]](s32)
+  ; RV32IF-NEXT:   $x12 = COPY [[C2]](s32)
+  ; RV32IF-NEXT:   $x13 = COPY [[COPY]](s32)
+  ; RV32IF-NEXT:   $x14 = COPY [[C3]](s32)
+  ; RV32IF-NEXT:   $x15 = COPY [[C4]](s32)
+  ; RV32IF-NEXT:   $x16 = COPY [[C5]](s32)
+  ; RV32IF-NEXT:   $x17 = COPY [[C6]](s32)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack1, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $f10_f, implicit-def $f10_f
+  ; RV32IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT1]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: caller_half_return_stack1
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $x10, $f10_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32IZFH-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32IZFH-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; RV32IZFH-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; RV32IZFH-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; RV32IZFH-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; RV32IZFH-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV32IZFH-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   $x10 = COPY [[C]](s32)
+  ; RV32IZFH-NEXT:   $x11 = COPY [[C1]](s32)
+  ; RV32IZFH-NEXT:   $x12 = COPY [[C2]](s32)
+  ; RV32IZFH-NEXT:   $x13 = COPY [[COPY]](s32)
+  ; RV32IZFH-NEXT:   $x14 = COPY [[C3]](s32)
+  ; RV32IZFH-NEXT:   $x15 = COPY [[C4]](s32)
+  ; RV32IZFH-NEXT:   $x16 = COPY [[C5]](s32)
+  ; RV32IZFH-NEXT:   $x17 = COPY [[C6]](s32)
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack1, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit $f10_h, implicit-def $f10_h
+  ; RV32IZFH-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   [[COPY2:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY2]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  %y = call half @callee_half_return_stack1(i32 0, i32 1, i32 2, i32 %v1, i32 5, i32 6, i32 7, i32 8, half %x)
+  ret half %y
+}
+
+define half @callee_half_return_stack2(half %v1, half %v2, half %v3, half %v4, half %v5, half %v6, half %v7, half %v8, half %x) nounwind {
+  ; RV32I-LABEL: name: callee_half_return_stack2
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32I-NEXT:   [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
+  ; RV32I-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32I-NEXT:   [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY4]](s32)
+  ; RV32I-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32I-NEXT:   [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
+  ; RV32I-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32I-NEXT:   [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY6]](s32)
+  ; RV32I-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $x17
+  ; RV32I-NEXT:   [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[COPY7]](s32)
+  ; RV32I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s32) from %fixed-stack.0, align 16)
+  ; RV32I-NEXT:   [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC8]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: callee_half_return_stack2
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $x10, $f10_f, $f11_f, $f12_f, $f13_f, $f14_f, $f15_f, $f16_f, $f17_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f11_f
+  ; RV32IF-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f12_f
+  ; RV32IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32IF-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $f13_f
+  ; RV32IF-NEXT:   [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
+  ; RV32IF-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $f14_f
+  ; RV32IF-NEXT:   [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY4]](s32)
+  ; RV32IF-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $f15_f
+  ; RV32IF-NEXT:   [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
+  ; RV32IF-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $f16_f
+  ; RV32IF-NEXT:   [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY6]](s32)
+  ; RV32IF-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $f17_f
+  ; RV32IF-NEXT:   [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[COPY7]](s32)
+  ; RV32IF-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IF-NEXT:   [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC8]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: callee_half_return_stack2
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $x10, $f10_h, $f11_h, $f12_h, $f13_h, $f14_h, $f15_h, $f16_h, $f17_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s16) = COPY $f11_h
+  ; RV32IZFH-NEXT:   [[COPY2:%[0-9]+]]:_(s16) = COPY $f12_h
+  ; RV32IZFH-NEXT:   [[COPY3:%[0-9]+]]:_(s16) = COPY $f13_h
+  ; RV32IZFH-NEXT:   [[COPY4:%[0-9]+]]:_(s16) = COPY $f14_h
+  ; RV32IZFH-NEXT:   [[COPY5:%[0-9]+]]:_(s16) = COPY $f15_h
+  ; RV32IZFH-NEXT:   [[COPY6:%[0-9]+]]:_(s16) = COPY $f16_h
+  ; RV32IZFH-NEXT:   [[COPY7:%[0-9]+]]:_(s16) = COPY $f17_h
+  ; RV32IZFH-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32IZFH-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[TRUNC]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  ret half %x
+}
+
+define half @caller_half_return_stack2(half %x, half %y) nounwind {
+  ; RV32I-LABEL: name: caller_half_return_stack2
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3C00
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH4200
+  ; RV32I-NEXT:   ADJCALLSTACKDOWN 4, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
+  ; RV32I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV32I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32)
+  ; RV32I-NEXT:   G_STORE [[ANYEXT8]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[ANYEXT1]](s32)
+  ; RV32I-NEXT:   $x12 = COPY [[ANYEXT2]](s32)
+  ; RV32I-NEXT:   $x13 = COPY [[ANYEXT3]](s32)
+  ; RV32I-NEXT:   $x14 = COPY [[ANYEXT4]](s32)
+  ; RV32I-NEXT:   $x15 = COPY [[ANYEXT5]](s32)
+  ; RV32I-NEXT:   $x16 = COPY [[ANYEXT6]](s32)
+  ; RV32I-NEXT:   $x17 = COPY [[ANYEXT7]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; RV32I-NEXT:   ADJCALLSTACKUP 4, 0, implicit-def $x2, implicit $x2
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
+  ; RV32I-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT9]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  ;
+  ; RV32IF-LABEL: name: caller_half_return_stack2
+  ; RV32IF: bb.1 (%ir-block.0):
+  ; RV32IF-NEXT:   liveins: $f10_f, $f11_f
+  ; RV32IF-NEXT: {{  $}}
+  ; RV32IF-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; RV32IF-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f11_f
+  ; RV32IF-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32IF-NEXT:   [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3C00
+  ; RV32IF-NEXT:   [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH4200
+  ; RV32IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   $f11_f = COPY [[ANYEXT1]](s32)
+  ; RV32IF-NEXT:   $f12_f = COPY [[ANYEXT2]](s32)
+  ; RV32IF-NEXT:   $f13_f = COPY [[ANYEXT3]](s32)
+  ; RV32IF-NEXT:   $f14_f = COPY [[ANYEXT4]](s32)
+  ; RV32IF-NEXT:   $f15_f = COPY [[ANYEXT5]](s32)
+  ; RV32IF-NEXT:   $f16_f = COPY [[ANYEXT6]](s32)
+  ; RV32IF-NEXT:   $f17_f = COPY [[ANYEXT7]](s32)
+  ; RV32IF-NEXT:   $x10 = COPY [[ANYEXT8]](s32)
+  ; RV32IF-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_f, implicit $f11_f, implicit $f12_f, implicit $f13_f, implicit $f14_f, implicit $f15_f, implicit $f16_f, implicit $f17_f, implicit $x10, implicit-def $f10_f
+  ; RV32IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT9]](s32)
+  ; RV32IF-NEXT:   PseudoRET implicit $f10_f
+  ;
+  ; RV32IZFH-LABEL: name: caller_half_return_stack2
+  ; RV32IZFH: bb.1 (%ir-block.0):
+  ; RV32IZFH-NEXT:   liveins: $f10_h, $f11_h
+  ; RV32IZFH-NEXT: {{  $}}
+  ; RV32IZFH-NEXT:   [[COPY:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   [[COPY1:%[0-9]+]]:_(s16) = COPY $f11_h
+  ; RV32IZFH-NEXT:   [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3C00
+  ; RV32IZFH-NEXT:   [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH4200
+  ; RV32IZFH-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   $f11_h = COPY [[C]](s16)
+  ; RV32IZFH-NEXT:   $f12_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   $f13_h = COPY [[C1]](s16)
+  ; RV32IZFH-NEXT:   $f14_h = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   $f15_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   $f16_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   $f17_h = COPY [[COPY1]](s16)
+  ; RV32IZFH-NEXT:   $x10 = COPY [[COPY]](s16)
+  ; RV32IZFH-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_h, implicit $f11_h, implicit $f12_h, implicit $f13_h, implicit $f14_h, implicit $f15_h, implicit $f16_h, implicit $f17_h, implicit $x10, implicit-def $f10_h
+  ; RV32IZFH-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; RV32IZFH-NEXT:   [[COPY2:%[0-9]+]]:_(s16) = COPY $f10_h
+  ; RV32IZFH-NEXT:   $f10_h = COPY [[COPY2]](s16)
+  ; RV32IZFH-NEXT:   PseudoRET implicit $f10_h
+  %z = call half @callee_half_return_stack2(half %x, half 1.0, half %x, half 3.0, half %x, half %y, half %y, half %y, half %x)
+  ret half %z
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv32-libcall.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv32-libcall.mir
new file mode 100644
index 0000000000000000000000000000000000000000..06ebc61eea099d76f17ed397d03ce76b3f051390
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv32-libcall.mir
@@ -0,0 +1,395 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=-d -mattr=-f -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+
+---
+name:            fptosi_s1_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s1_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s1) = G_FPTOSI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s1)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            fptoui_s1_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s1_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s1) = G_FPTOUI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s1)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            fptosi_s8_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s8_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s8) = G_FPTOSI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s8)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s8_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s8_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s8) = G_FPTOUI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s8)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s16_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s16_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s16) = G_FPTOSI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s16)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s16_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s16_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s16) = G_FPTOUI %0:_(s32)
+    %2:_(s32) = G_ANYEXT %1:_(s16)
+    $x10 = COPY %2:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s32_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s32_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = G_FPTOSI %0:_(s32)
+    $x10 = COPY %1:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s32_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s32_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = G_FPTOUI %0:_(s32)
+    $x10 = COPY %1:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s1_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptosi_s1_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s1) = G_FPTOSI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s1)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s1_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptoui_s1_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s1) = G_FPTOUI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s1)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s8_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptosi_s8_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s8) = G_FPTOSI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s8)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            fptoui_s8_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptoui_s8_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s8) = G_FPTOUI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s8)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s16_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptosi_s16_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s16) = G_FPTOSI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s16)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s16_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptoui_s16_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s16) = G_FPTOUI %0:_(s64)
+    %4:_(s32) = G_ANYEXT %3:_(s16)
+    $x10 = COPY %4:_(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s32_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptosi_s32_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s32) = G_FPTOSI %0:_(s64)
+    $x10 = COPY %3:_(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            fptoui_s32_s64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: fptoui_s32_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1:_(s32), %2:_(s32)
+    %3:_(s32) = G_FPTOUI %0:_(s64)
+    $x10 = COPY %3:_(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv64-libcall.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv64-libcall.mir
new file mode 100644
index 0000000000000000000000000000000000000000..629ad24f8780b8005bb334061b2d8e454e14fe68
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fptoi-rv64-libcall.mir
@@ -0,0 +1,471 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=-d -mattr=-f -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+
+---
+name:            fptosi_s1_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s1_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s1) = G_FPTOSI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s1)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s1_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s1_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s1) = G_FPTOUI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s1)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s8_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s8_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s8) = G_FPTOSI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s8)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s8_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s8_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s8) = G_FPTOUI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s8)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s16_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s16_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s16) = G_FPTOSI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s16)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s16_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s16_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s16) = G_FPTOUI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s16)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s32_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s32_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s32) = G_FPTOSI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s32)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s32_s32
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s32_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1:_(s64)
+    %2:_(s32) = G_FPTOUI %0:_(s32)
+    %3:_(s64) = G_ANYEXT %2:_(s32)
+    $x10 = COPY %3:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s64_s32
+body:             |
+  bb.1:
+    liveins: $f10_f
+
+    ; CHECK-LABEL: name: fptosi_s64_s32
+    ; CHECK: liveins: $f10_f
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixsfdi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $f10_f
+    %1:_(s64) = G_FPTOSI %0(s32)
+    $x10 = COPY %1(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s64_s32
+body:             |
+  bb.1:
+    liveins: $f10_f
+
+    ; CHECK-LABEL: name: fptoui_s64_s32
+    ; CHECK: liveins: $f10_f
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $f10_f
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunssfdi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $f10_f
+    %1:_(s64) = G_FPTOUI %0(s32)
+    $x10 = COPY %1(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s1_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s1_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s1) = G_FPTOSI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s1)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s1_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s1_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s1) = G_FPTOUI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s1)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s8_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s8_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s8) = G_FPTOSI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s8)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s8_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s8_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s8) = G_FPTOUI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s8)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s16_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s16_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s16) = G_FPTOSI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s16)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s16_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s16_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s16) = G_FPTOUI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s16)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s32_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s32_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s32) = G_FPTOSI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s32)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s32_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s32_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfsi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s32) = G_FPTOUI %0:_(s64)
+    %2:_(s64) = G_ANYEXT %1:_(s32)
+    $x10 = COPY %2:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptosi_s64_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptosi_s64_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixdfdi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = G_FPTOSI %0:_(s64)
+    $x10 = COPY %1:_(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            fptoui_s64_s64
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: fptoui_s64_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__fixunsdfdi, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = G_FPTOUI %0:_(s64)
+    $x10 = COPY %1:_(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir
index 2e4a39c468111f265f73a35661ad19118be6a41b..46a7df44955820a80f549381b28ef01232731dba 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir
@@ -68,12 +68,14 @@ body:             |
     ; RV32: liveins: $x10
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
-    ; RV32-NEXT: $x10 = COPY [[COPY]](s32)
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; RV32-NEXT: $x10 = COPY [[AND]](s32)
     ; RV32-NEXT: PseudoRET implicit $x10
     %0:_(s32) = COPY $x10
-    %1:_(s64) = G_ZEXT %0(s32)
-    %2:_(s32), %3:_(s32) = G_UNMERGE_VALUES %1(s64)
-    $x10 = COPY %2(s32)
+    %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0(s32)
+    %4:_(s32) = G_ZEXT %2(s16)
+    $x10 = COPY %4(s32)
     PseudoRET implicit $x10
 ...
 ---
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll
index d945cf561698128adec1b757692c7e9836f8733e..7a96aad31f084305faf20c93caeeb8e6bf3794b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 define <2 x half> @select_v2f16(i1 zeroext %c, <2 x half> %a, <2 x half> %b) {
@@ -343,3 +343,123 @@ define <16 x double> @selectcc_v16f64(double %a, double %b, <16 x double> %c, <1
   %v = select i1 %cmp, <16 x double> %c, <16 x double> %d
   ret <16 x double> %v
 }
+
+define <2 x bfloat> @select_v2bf16(i1 zeroext %c, <2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: select_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b
+  ret <2 x bfloat> %v
+}
+
+define <2 x bfloat> @selectcc_v2bf16(bfloat %a, bfloat %b, <2 x bfloat> %c, <2 x bfloat> %d) {
+; CHECK-LABEL: selectcc_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <2 x bfloat> %c, <2 x bfloat> %d
+  ret <2 x bfloat> %v
+}
+
+define <4 x bfloat> @select_v4bf16(i1 zeroext %c, <4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: select_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <4 x bfloat> %a, <4 x bfloat> %b
+  ret <4 x bfloat> %v
+}
+
+define <4 x bfloat> @selectcc_v4bf16(bfloat %a, bfloat %b, <4 x bfloat> %c, <4 x bfloat> %d) {
+; CHECK-LABEL: selectcc_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <4 x bfloat> %c, <4 x bfloat> %d
+  ret <4 x bfloat> %v
+}
+
+define <8 x bfloat> @select_v8bf16(i1 zeroext %c, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: select_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <8 x bfloat> %a, <8 x bfloat> %b
+  ret <8 x bfloat> %v
+}
+
+define <8 x bfloat> @selectcc_v8bf16(bfloat %a, bfloat %b, <8 x bfloat> %c, <8 x bfloat> %d) {
+; CHECK-LABEL: selectcc_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <8 x bfloat> %c, <8 x bfloat> %d
+  ret <8 x bfloat> %v
+}
+
+define <16 x bfloat> @select_v16bf16(i1 zeroext %c, <16 x bfloat> %a, <16 x bfloat> %b) {
+; CHECK-LABEL: select_v16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <16 x bfloat> %a, <16 x bfloat> %b
+  ret <16 x bfloat> %v
+}
+
+define <16 x bfloat> @selectcc_v16bf16(bfloat %a, bfloat %b, <16 x bfloat> %c, <16 x bfloat> %d) {
+; CHECK-LABEL: selectcc_v16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <16 x bfloat> %c, <16 x bfloat> %d
+  ret <16 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
index 9f0561b394b819e3666d0757da364bf1f3bc143b..d360c3f635b5c3c003b5c99615d5854358a9c118 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV64
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV32ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV64ZVFHMIN
 
 declare <4 x i1> @llvm.vp.merge.v4i1(<4 x i1>, <4 x i1>, <4 x i1>, i32)
@@ -1240,3 +1240,139 @@ define <32 x double> @vpmerge_vf_v32f64(double %a, <32 x double> %vb, <32 x i1>
   %v = call <32 x double> @llvm.vp.merge.v32f64(<32 x i1> %m, <32 x double> %va, <32 x double> %vb, i32 %evl)
   ret <32 x double> %v
 }
+
+declare <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32)
+
+define <2 x bfloat> @vpmerge_vv_v2bf16(<2 x bfloat> %va, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl)
+  ret <2 x bfloat> %v
+}
+
+define <2 x bfloat> @vpmerge_vf_v2bf16(bfloat %a, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) {
+; ZVFH-LABEL: vpmerge_vf_v2bf16:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
+; ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vpmerge_vf_v2bf16:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <2 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <2 x bfloat> %elt.head, <2 x bfloat> poison, <2 x i32> zeroinitializer
+  %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl)
+  ret <2 x bfloat> %v
+}
+
+declare <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32)
+
+define <4 x bfloat> @vpmerge_vv_v4bf16(<4 x bfloat> %va, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl)
+  ret <4 x bfloat> %v
+}
+
+define <4 x bfloat> @vpmerge_vf_v4bf16(bfloat %a, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) {
+; ZVFH-LABEL: vpmerge_vf_v4bf16:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
+; ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vpmerge_vf_v4bf16:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <4 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <4 x bfloat> %elt.head, <4 x bfloat> poison, <4 x i32> zeroinitializer
+  %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl)
+  ret <4 x bfloat> %v
+}
+
+declare <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32)
+
+define <8 x bfloat> @vpmerge_vv_v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl)
+  ret <8 x bfloat> %v
+}
+
+define <8 x bfloat> @vpmerge_vf_v8bf16(bfloat %a, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) {
+; ZVFH-LABEL: vpmerge_vf_v8bf16:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vpmerge_vf_v8bf16:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v10, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v10, v0.t
+; ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <8 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <8 x bfloat> %elt.head, <8 x bfloat> poison, <8 x i32> zeroinitializer
+  %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl)
+  ret <8 x bfloat> %v
+}
+
+declare <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32)
+
+define <16 x bfloat> @vpmerge_vv_v16bf16(<16 x bfloat> %va, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_v16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
+; CHECK-NEXT:    vmerge.vvm v10, v10, v8, v0
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl)
+  ret <16 x bfloat> %v
+}
+
+define <16 x bfloat> @vpmerge_vf_v16bf16(bfloat %a, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) {
+; ZVFH-LABEL: vpmerge_vf_v16bf16:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
+; ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vpmerge_vf_v16bf16:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v12, v0.t
+; ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <16 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <16 x bfloat> %elt.head, <16 x bfloat> poison, <16 x i32> zeroinitializer
+  %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl)
+  ret <16 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
index 0a2ed3eb1ffbf7b46e414d0e29239aab35999e24..c5d9cdacae749f1b81523373c33a6b79e7fe2b0e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare <1 x i1> @llvm.vp.select.v1i1(<1 x i1>, <1 x i1>, <1 x i1>, i32)
@@ -683,3 +683,51 @@ define <16 x double> @select_v16f64(<16 x i1> %a, <16 x double> %b, <16 x double
   %v = call <16 x double> @llvm.vp.select.v16f64(<16 x i1> %a, <16 x double> %b, <16 x double> %c, i32 %evl)
   ret <16 x double> %v
 }
+
+declare <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32)
+
+define <2 x bfloat> @select_v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 %evl)
+  ret <2 x bfloat> %v
+}
+
+declare <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32)
+
+define <4 x bfloat> @select_v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 %evl)
+  ret <4 x bfloat> %v
+}
+
+declare <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32)
+
+define <8 x bfloat> @select_v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 %evl)
+  ret <8 x bfloat> %v
+}
+
+declare <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32)
+
+define <16 x bfloat> @select_v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_v16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 %evl)
+  ret <16 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
index f8581d8e21b390cc7bc0036a1de0c392da0b4c48..2b9d847a9e873bb116a054172225b9c719ab4079 100644
--- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @select_nxv1f16(i1 zeroext %c, <vscale x 1 x half> %a, <vscale x 1 x half> %b) {
@@ -427,3 +427,183 @@ define <vscale x 8 x double> @selectcc_nxv8f64(double %a, double %b, <vscale x 8
   %v = select i1 %cmp, <vscale x 8 x double> %c, <vscale x 8 x double> %d
   ret <vscale x 8 x double> %v
 }
+
+define <vscale x 1 x bfloat> @select_nxv1bf16(i1 zeroext %c, <vscale x 1 x bfloat> %a, <vscale x 1 x bfloat> %b) {
+; CHECK-LABEL: select_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 1 x bfloat> %a, <vscale x 1 x bfloat> %b
+  ret <vscale x 1 x bfloat> %v
+}
+
+define <vscale x 1 x bfloat> @selectcc_nxv1bf16(bfloat %a, bfloat %b, <vscale x 1 x bfloat> %c, <vscale x 1 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 1 x bfloat> %c, <vscale x 1 x bfloat> %d
+  ret <vscale x 1 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @select_nxv2bf16(i1 zeroext %c, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: select_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @selectcc_nxv2bf16(bfloat %a, bfloat %b, <vscale x 2 x bfloat> %c, <vscale x 2 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 2 x bfloat> %c, <vscale x 2 x bfloat> %d
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 4 x bfloat> @select_nxv4bf16(i1 zeroext %c, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: select_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b
+  ret <vscale x 4 x bfloat> %v
+}
+
+define <vscale x 4 x bfloat> @selectcc_nxv4bf16(bfloat %a, bfloat %b, <vscale x 4 x bfloat> %c, <vscale x 4 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 4 x bfloat> %c, <vscale x 4 x bfloat> %d
+  ret <vscale x 4 x bfloat> %v
+}
+
+define <vscale x 8 x bfloat> @select_nxv8bf16(i1 zeroext %c, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: select_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b
+  ret <vscale x 8 x bfloat> %v
+}
+
+define <vscale x 8 x bfloat> @selectcc_nxv8bf16(bfloat %a, bfloat %b, <vscale x 8 x bfloat> %c, <vscale x 8 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 8 x bfloat> %c, <vscale x 8 x bfloat> %d
+  ret <vscale x 8 x bfloat> %v
+}
+
+define <vscale x 16 x bfloat> @select_nxv16bf16(i1 zeroext %c, <vscale x 16 x bfloat> %a, <vscale x 16 x bfloat> %b) {
+; CHECK-LABEL: select_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.x v16, a0
+; CHECK-NEXT:    vmsne.vi v0, v16, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 16 x bfloat> %a, <vscale x 16 x bfloat> %b
+  ret <vscale x 16 x bfloat> %v
+}
+
+define <vscale x 16 x bfloat> @selectcc_nxv16bf16(bfloat %a, bfloat %b, <vscale x 16 x bfloat> %c, <vscale x 16 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.x v16, a0
+; CHECK-NEXT:    vmsne.vi v0, v16, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 16 x bfloat> %c, <vscale x 16 x bfloat> %d
+  ret <vscale x 16 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @select_nxv32bf16(i1 zeroext %c, <vscale x 32 x bfloat> %a, <vscale x 32 x bfloat> %b) {
+; CHECK-LABEL: select_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vmv.v.x v24, a0
+; CHECK-NEXT:    vmsne.vi v0, v24, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
+; CHECK-NEXT:    ret
+  %v = select i1 %c, <vscale x 32 x bfloat> %a, <vscale x 32 x bfloat> %b
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @selectcc_nxv32bf16(bfloat %a, bfloat %b, <vscale x 32 x bfloat> %c, <vscale x 32 x bfloat> %d) {
+; CHECK-LABEL: selectcc_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.s.bf16 fa5, fa1
+; CHECK-NEXT:    fcvt.s.bf16 fa4, fa0
+; CHECK-NEXT:    feq.s a0, fa4, fa5
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vmv.v.x v24, a0
+; CHECK-NEXT:    vmsne.vi v0, v24, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq bfloat %a, %b
+  %v = select i1 %cmp, <vscale x 32 x bfloat> %c, <vscale x 32 x bfloat> %d
+  ret <vscale x 32 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..28d7588b9347a791c131b0506f0ba0a8e0ad97e0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
@@ -0,0 +1,210 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+define <4 x i8> @test_v4i16_v4i8(<4 x i16> %x) {
+; CHECK-LABEL: test_v4i16_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i16> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i16>
+  %c = icmp ult <4 x i16> %x, splat (i16 256)
+  %d = select <4 x i1> %c, <4 x i16> %x, <4 x i16> %b
+  %e = trunc <4 x i16> %d to <4 x i8>
+  ret <4 x i8> %e
+}
+
+define <4 x i8> @test_v4i32_v4i8(<4 x i32> %x) {
+; CHECK-LABEL: test_v4i32_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i32> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i32>
+  %c = icmp ult <4 x i32> %x, splat (i32 256)
+  %d = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %b
+  %e = trunc <4 x i32> %d to <4 x i8>
+  ret <4 x i8> %e
+}
+
+define <4 x i8> @test_v4i64_v4i8(<4 x i64> %x) {
+; CHECK-LABEL: test_v4i64_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v10, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i64> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i64>
+  %c = icmp ult <4 x i64> %x, splat (i64 256)
+  %d = select <4 x i1> %c, <4 x i64> %x, <4 x i64> %b
+  %e = trunc <4 x i64> %d to <4 x i8>
+  ret <4 x i8> %e
+}
+
+define <4 x i16> @test_v4i32_v4i16(<4 x i32> %x) {
+; CHECK-LABEL: test_v4i32_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i32> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i32>
+  %c = icmp ult <4 x i32> %x, splat (i32 65536)
+  %d = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %b
+  %e = trunc <4 x i32> %d to <4 x i16>
+  ret <4 x i16> %e
+}
+
+define <4 x i16> @test_v4i64_v4i16(<4 x i64> %x) {
+; CHECK-LABEL: test_v4i64_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v10, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i64> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i64>
+  %c = icmp ult <4 x i64> %x, splat (i64 65536)
+  %d = select <4 x i1> %c, <4 x i64> %x, <4 x i64> %b
+  %e = trunc <4 x i64> %d to <4 x i16>
+  ret <4 x i16> %e
+}
+
+define <4 x i32> @test_v4i64_v4i32(<4 x i64> %x) {
+; CHECK-LABEL: test_v4i64_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <4 x i64> %x, zeroinitializer
+  %b = sext <4 x i1> %a to <4 x i64>
+  %c = icmp ult <4 x i64> %x, splat (i64 4294967296)
+  %d = select <4 x i1> %c, <4 x i64> %x, <4 x i64> %b
+  %e = trunc <4 x i64> %d to <4 x i32>
+  ret <4 x i32> %e
+}
+
+define <vscale x 4 x i8> @test_nxv4i16_nxv4i8(<vscale x 4 x i16> %x) {
+; CHECK-LABEL: test_nxv4i16_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i16> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i16>
+  %c = icmp ult <vscale x 4 x i16> %x, splat (i16 256)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i16> %x, <vscale x 4 x i16> %b
+  %e = trunc <vscale x 4 x i16> %d to <vscale x 4 x i8>
+  ret <vscale x 4 x i8> %e
+}
+
+define <vscale x 4 x i8> @test_nxv4i32_nxv4i8(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_nxv4i32_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v10, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i32> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i32>
+  %c = icmp ult <vscale x 4 x i32> %x, splat (i32 256)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i32> %x, <vscale x 4 x i32> %b
+  %e = trunc <vscale x 4 x i32> %d to <vscale x 4 x i8>
+  ret <vscale x 4 x i8> %e
+}
+
+define <vscale x 4 x i8> @test_nxv4i64_nxv4i8(<vscale x 4 x i64> %x) {
+; CHECK-LABEL: test_nxv4i64_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v12, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i64> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i64>
+  %c = icmp ult <vscale x 4 x i64> %x, splat (i64 256)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i64> %x, <vscale x 4 x i64> %b
+  %e = trunc <vscale x 4 x i64> %d to <vscale x 4 x i8>
+  ret <vscale x 4 x i8> %e
+}
+
+define <vscale x 4 x i16> @test_nxv4i32_nxv4i16(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_nxv4i32_nxv4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i32> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i32>
+  %c = icmp ult <vscale x 4 x i32> %x, splat (i32 65536)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i32> %x, <vscale x 4 x i32> %b
+  %e = trunc <vscale x 4 x i32> %d to <vscale x 4 x i16>
+  ret <vscale x 4 x i16> %e
+}
+
+define <vscale x 4 x i16> @test_nxv4i64_nxv4i16(<vscale x 4 x i64> %x) {
+; CHECK-LABEL: test_nxv4i64_nxv4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v12, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i64> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i64>
+  %c = icmp ult <vscale x 4 x i64> %x, splat (i64 65536)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i64> %x, <vscale x 4 x i64> %b
+  %e = trunc <vscale x 4 x i64> %d to <vscale x 4 x i16>
+  ret <vscale x 4 x i16> %e
+}
+
+define <vscale x 4 x i32> @test_nxv4i64_nxv4i32(<vscale x 4 x i64> %x) {
+; CHECK-LABEL: test_nxv4i64_nxv4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v12, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a = icmp sgt <vscale x 4 x i64> %x, zeroinitializer
+  %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i64>
+  %c = icmp ult <vscale x 4 x i64> %x, splat (i64 4294967296)
+  %d = select <vscale x 4 x i1> %c, <vscale x 4 x i64> %x, <vscale x 4 x i64> %b
+  %e = trunc <vscale x 4 x i64> %d to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %e
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
index 14a1f084c3985be7c39aeedbf55d4f1d4d51ee3f..d1f344d52763db8d31d056db6b604415a33162b9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
@@ -33,7 +33,7 @@ define <vscale x 1 x i1> @intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -75,7 +75,7 @@ define <vscale x 2 x i1> @intrinsic_vmsbf_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -117,7 +117,7 @@ define <vscale x 4 x i1> @intrinsic_vmsbf_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -159,9 +159,9 @@ define <vscale x 8 x i1> @intrinsic_vmsbf_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsbf.mask.nxv8i1(
@@ -201,7 +201,7 @@ define <vscale x 16 x i1> @intrinsic_vmsbf_mask_m_nxv16i1_nxv16i1(<vscale x 16 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -243,7 +243,7 @@ define <vscale x 32 x i1> @intrinsic_vmsbf_mask_m_nxv32i1_nxv32i1(<vscale x 32 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -285,7 +285,7 @@ define <vscale x 64 x i1> @intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1(<vscale x 64 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsif.ll b/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
index 05d402afc934cdb32f85f8bbd64a715df655f800..1dc52eb55455bac46084d39c1b713c5eba7383d8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
@@ -33,7 +33,7 @@ define <vscale x 1 x i1> @intrinsic_vmsif_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -75,7 +75,7 @@ define <vscale x 2 x i1> @intrinsic_vmsif_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -117,7 +117,7 @@ define <vscale x 4 x i1> @intrinsic_vmsif_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -159,9 +159,9 @@ define <vscale x 8 x i1> @intrinsic_vmsif_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsif.mask.nxv8i1(
@@ -201,7 +201,7 @@ define <vscale x 16 x i1> @intrinsic_vmsif_mask_m_nxv16i1_nxv16i1(<vscale x 16 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -243,7 +243,7 @@ define <vscale x 32 x i1> @intrinsic_vmsif_mask_m_nxv32i1_nxv32i1(<vscale x 32 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -285,7 +285,7 @@ define <vscale x 64 x i1> @intrinsic_vmsif_mask_m_nxv64i1_nxv64i1(<vscale x 64 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsof.ll b/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
index 0c60681ea8de0da5c8962524b60b7c7017f792bd..b0a28e6e455b07cd445f30556d6b2fab3d8edfd8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
@@ -33,7 +33,7 @@ define <vscale x 1 x i1> @intrinsic_vmsof_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -75,7 +75,7 @@ define <vscale x 2 x i1> @intrinsic_vmsof_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -117,7 +117,7 @@ define <vscale x 4 x i1> @intrinsic_vmsof_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -159,9 +159,9 @@ define <vscale x 8 x i1> @intrinsic_vmsof_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1>
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsof.mask.nxv8i1(
@@ -201,7 +201,7 @@ define <vscale x 16 x i1> @intrinsic_vmsof_mask_m_nxv16i1_nxv16i1(<vscale x 16 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -243,7 +243,7 @@ define <vscale x 32 x i1> @intrinsic_vmsof_mask_m_nxv32i1_nxv32i1(<vscale x 32 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -285,7 +285,7 @@ define <vscale x 64 x i1> @intrinsic_vmsof_mask_m_nxv64i1_nxv64i1(<vscale x 64 x
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, mu
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
index 094e6c9cc754fa3a8ae1ef1ef33c60fea78222e1..e33c795169fab8d5a5edca40ea744af4f4d427db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFHMIN
 declare <vscale x 1 x i1> @llvm.vp.merge.nxv1i1(<vscale x 1 x i1>, <vscale x 1 x i1>, <vscale x 1 x i1>, i32)
 
@@ -1547,3 +1547,303 @@ define <vscale x 8 x double> @vpmerge_vf_nxv8f64(double %a, <vscale x 8 x double
   %v = call <vscale x 8 x double> @llvm.vp.merge.nxv8f64(<vscale x 8 x i1> %m, <vscale x 8 x double> %va, <vscale x 8 x double> %vb, i32 %evl)
   ret <vscale x 8 x double> %v
 }
+
+declare <vscale x 1 x bfloat> @llvm.vp.merge.nxv1bf16(<vscale x 1 x i1>, <vscale x 1 x bfloat>, <vscale x 1 x bfloat>, i32)
+
+define <vscale x 1 x bfloat> @vpmerge_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x bfloat> @llvm.vp.merge.nxv1bf16(<vscale x 1 x i1> %m, <vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, i32 %evl)
+  ret <vscale x 1 x bfloat> %v
+}
+
+define <vscale x 1 x bfloat> @vpmerge_vf_nxv1bf16(bfloat %a, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv1bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv1bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
+  %v = call <vscale x 1 x bfloat> @llvm.vp.merge.nxv1bf16(<vscale x 1 x i1> %m, <vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, i32 %evl)
+  ret <vscale x 1 x bfloat> %v
+}
+
+declare <vscale x 2 x bfloat> @llvm.vp.merge.nxv2bf16(<vscale x 2 x i1>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>, i32)
+
+define <vscale x 2 x bfloat> @vpmerge_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.merge.nxv2bf16(<vscale x 2 x i1> %m, <vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 %evl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @vpmerge_vf_nxv2bf16(bfloat %a, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv2bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv2bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 2 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 2 x bfloat> %elt.head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer
+  %v = call <vscale x 2 x bfloat> @llvm.vp.merge.nxv2bf16(<vscale x 2 x i1> %m, <vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 %evl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+declare <vscale x 4 x bfloat> @llvm.vp.merge.nxv4bf16(<vscale x 4 x i1>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>, i32)
+
+define <vscale x 4 x bfloat> @vpmerge_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x bfloat> @llvm.vp.merge.nxv4bf16(<vscale x 4 x i1> %m, <vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, i32 %evl)
+  ret <vscale x 4 x bfloat> %v
+}
+
+define <vscale x 4 x bfloat> @vpmerge_vf_nxv4bf16(bfloat %a, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv4bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv4bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v10, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v10, v0.t
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v10, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v10, v0.t
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 4 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 4 x bfloat> %elt.head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x bfloat> @llvm.vp.merge.nxv4bf16(<vscale x 4 x i1> %m, <vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, i32 %evl)
+  ret <vscale x 4 x bfloat> %v
+}
+
+declare <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
+
+define <vscale x 8 x bfloat> @vpmerge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
+; CHECK-NEXT:    vmerge.vvm v10, v10, v8, v0
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> %m, <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, i32 %evl)
+  ret <vscale x 8 x bfloat> %v
+}
+
+define <vscale x 8 x bfloat> @vpmerge_vf_nxv8bf16(bfloat %a, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv8bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv8bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v12, v0.t
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v12, v0.t
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 8 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 8 x bfloat> %elt.head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
+  %v = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> %m, <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, i32 %evl)
+  ret <vscale x 8 x bfloat> %v
+}
+
+declare <vscale x 16 x bfloat> @llvm.vp.merge.nxv16bf16(<vscale x 16 x i1>, <vscale x 16 x bfloat>, <vscale x 16 x bfloat>, i32)
+
+define <vscale x 16 x bfloat> @vpmerge_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, <vscale x 16 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
+; CHECK-NEXT:    vmerge.vvm v12, v12, v8, v0
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x bfloat> @llvm.vp.merge.nxv16bf16(<vscale x 16 x i1> %m, <vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, i32 %evl)
+  ret <vscale x 16 x bfloat> %v
+}
+
+define <vscale x 16 x bfloat> @vpmerge_vf_nxv16bf16(bfloat %a, <vscale x 16 x bfloat> %vb, <vscale x 16 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv16bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv16bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v16, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v16, v0.t
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v16, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v16, v0.t
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 16 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 16 x bfloat> %elt.head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer
+  %v = call <vscale x 16 x bfloat> @llvm.vp.merge.nxv16bf16(<vscale x 16 x i1> %m, <vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, i32 %evl)
+  ret <vscale x 16 x bfloat> %v
+}
+
+declare <vscale x 32 x bfloat> @llvm.vp.merge.nxv32bf16(<vscale x 32 x i1>, <vscale x 32 x bfloat>, <vscale x 32 x bfloat>, i32)
+
+define <vscale x 32 x bfloat> @vpmerge_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, <vscale x 32 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vpmerge_vv_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    vmv8r.v v8, v16
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vp.merge.nxv32bf16(<vscale x 32 x i1> %m, <vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, i32 %evl)
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @vpmerge_vf_nxv32bf16(bfloat %a, <vscale x 32 x bfloat> %vb, <vscale x 32 x i1> %m, i32 zeroext %evl) {
+; RV32ZVFH-LABEL: vpmerge_vf_nxv32bf16:
+; RV32ZVFH:       # %bb.0:
+; RV32ZVFH-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
+; RV32ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32ZVFH-NEXT:    ret
+;
+; RV64ZVFH-LABEL: vpmerge_vf_nxv32bf16:
+; RV64ZVFH:       # %bb.0:
+; RV64ZVFH-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
+; RV64ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64ZVFH-NEXT:    ret
+;
+; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16:
+; RV32ZVFHMIN:       # %bb.0:
+; RV32ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV32ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32ZVFHMIN-NEXT:    vfmv.v.f v24, fa5
+; RV32ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; RV32ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v16, v24
+; RV32ZVFHMIN-NEXT:    vmv.v.v v20, v16
+; RV32ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
+; RV32ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v16, v0
+; RV32ZVFHMIN-NEXT:    ret
+;
+; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16:
+; RV64ZVFHMIN:       # %bb.0:
+; RV64ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; RV64ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV64ZVFHMIN-NEXT:    vfmv.v.f v24, fa5
+; RV64ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; RV64ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v16, v24
+; RV64ZVFHMIN-NEXT:    vmv.v.v v20, v16
+; RV64ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
+; RV64ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v16, v0
+; RV64ZVFHMIN-NEXT:    ret
+  %elt.head = insertelement <vscale x 32 x bfloat> poison, bfloat %a, i32 0
+  %va = shufflevector <vscale x 32 x bfloat> %elt.head, <vscale x 32 x bfloat> poison, <vscale x 32 x i32> zeroinitializer
+  %v = call <vscale x 32 x bfloat> @llvm.vp.merge.nxv32bf16(<vscale x 32 x i1> %m, <vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, i32 %evl)
+  ret <vscale x 32 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
index 53b8e4a78b756f4673dde66138ac746d7f56220d..ee7a94cf5e4971201393fb4b39a70a5f33b4a3df 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN
 
 define <vscale x 1 x half> @vfmerge_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %cond) {
@@ -512,3 +512,219 @@ define void @vselect_legalize_regression(<vscale x 16 x double> %a, <vscale x 16
   store <vscale x 16 x double> %sel, ptr %out
   ret void
 }
+
+define <vscale x 1 x bfloat> @vfmerge_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 1 x i1> %cond, <vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb
+  ret <vscale x 1 x bfloat> %vc
+}
+
+define <vscale x 1 x bfloat> @vfmerge_fv_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat %b, <vscale x 1 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv1bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv1bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 1 x bfloat> %head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
+  %vc = select <vscale x 1 x i1> %cond, <vscale x 1 x bfloat> %splat, <vscale x 1 x bfloat> %va
+  ret <vscale x 1 x bfloat> %vc
+}
+
+define <vscale x 2 x bfloat> @vfmerge_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 2 x i1> %cond, <vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb
+  ret <vscale x 2 x bfloat> %vc
+}
+
+define <vscale x 2 x bfloat> @vfmerge_fv_nxv2bf16(<vscale x 2 x bfloat> %va, bfloat %b, <vscale x 2 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv2bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv2bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v9, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 2 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 2 x bfloat> %head, <vscale x 2 x bfloat> poison, <vscale x 2 x i32> zeroinitializer
+  %vc = select <vscale x 2 x i1> %cond, <vscale x 2 x bfloat> %splat, <vscale x 2 x bfloat> %va
+  ret <vscale x 2 x bfloat> %vc
+}
+
+define <vscale x 4 x bfloat> @vfmerge_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 4 x i1> %cond, <vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb
+  ret <vscale x 4 x bfloat> %vc
+}
+
+define <vscale x 4 x bfloat> @vfmerge_fv_nxv4bf16(<vscale x 4 x bfloat> %va, bfloat %b, <vscale x 4 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv4bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv4bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v10, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v10, v0.t
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 4 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 4 x bfloat> %head, <vscale x 4 x bfloat> poison, <vscale x 4 x i32> zeroinitializer
+  %vc = select <vscale x 4 x i1> %cond, <vscale x 4 x bfloat> %splat, <vscale x 4 x bfloat> %va
+  ret <vscale x 4 x bfloat> %vc
+}
+
+define <vscale x 8 x bfloat> @vfmerge_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 8 x i1> %cond, <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb
+  ret <vscale x 8 x bfloat> %vc
+}
+
+define <vscale x 8 x bfloat> @vfmerge_fv_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat %b, <vscale x 8 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv8bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv8bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v12, v0.t
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 8 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 8 x bfloat> %head, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
+  %vc = select <vscale x 8 x i1> %cond, <vscale x 8 x bfloat> %splat, <vscale x 8 x bfloat> %va
+  ret <vscale x 8 x bfloat> %vc
+}
+
+define <vscale x 8 x bfloat> @vfmerge_zv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %cond) {
+; CHECK-LABEL: vfmerge_zv_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v8, v8, 0, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 8 x i1> %cond, <vscale x 8 x bfloat> splat (bfloat zeroinitializer), <vscale x 8 x bfloat> %va
+  ret <vscale x 8 x bfloat> %vc
+}
+
+define <vscale x 8 x bfloat> @vmerge_truelhs_nxv8bf16_0(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb) {
+; CHECK-LABEL: vmerge_truelhs_nxv8bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 8 x i1> splat (i1 1), <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb
+  ret <vscale x 8 x bfloat> %vc
+}
+
+define <vscale x 8 x bfloat> @vmerge_falselhs_nxv8bf16_0(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb) {
+; CHECK-LABEL: vmerge_falselhs_nxv8bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 8 x i1> zeroinitializer, <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb
+  ret <vscale x 8 x bfloat> %vc
+}
+
+define <vscale x 16 x bfloat> @vfmerge_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb, <vscale x 16 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 16 x i1> %cond, <vscale x 16 x bfloat> %va, <vscale x 16 x bfloat> %vb
+  ret <vscale x 16 x bfloat> %vc
+}
+
+define <vscale x 16 x bfloat> @vfmerge_fv_nxv16bf16(<vscale x 16 x bfloat> %va, bfloat %b, <vscale x 16 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv16bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv16bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v16, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v8, v16, v0.t
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 16 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 16 x bfloat> %head, <vscale x 16 x bfloat> poison, <vscale x 16 x i32> zeroinitializer
+  %vc = select <vscale x 16 x i1> %cond, <vscale x 16 x bfloat> %splat, <vscale x 16 x bfloat> %va
+  ret <vscale x 16 x bfloat> %vc
+}
+
+define <vscale x 32 x bfloat> @vfmerge_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb, <vscale x 32 x i1> %cond) {
+; CHECK-LABEL: vfmerge_vv_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
+; CHECK-NEXT:    ret
+  %vc = select <vscale x 32 x i1> %cond, <vscale x 32 x bfloat> %va, <vscale x 32 x bfloat> %vb
+  ret <vscale x 32 x bfloat> %vc
+}
+
+define <vscale x 32 x bfloat> @vfmerge_fv_nxv32bf16(<vscale x 32 x bfloat> %va, bfloat %b, <vscale x 32 x i1> %cond) {
+; CHECK-ZVFH-LABEL: vfmerge_fv_nxv32bf16:
+; CHECK-ZVFH:       # %bb.0:
+; CHECK-ZVFH-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-ZVFH-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; CHECK-ZVFH-NEXT:    ret
+;
+; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv32bf16:
+; CHECK-ZVFHMIN:       # %bb.0:
+; CHECK-ZVFHMIN-NEXT:    fcvt.s.bf16 fa5, fa0
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfmv.v.f v16, fa5
+; CHECK-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vfncvtbf16.f.f.w v24, v16
+; CHECK-ZVFHMIN-NEXT:    vmv.v.v v28, v24
+; CHECK-ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v24, v0
+; CHECK-ZVFHMIN-NEXT:    ret
+  %head = insertelement <vscale x 32 x bfloat> poison, bfloat %b, i32 0
+  %splat = shufflevector <vscale x 32 x bfloat> %head, <vscale x 32 x bfloat> poison, <vscale x 32 x i32> zeroinitializer
+  %vc = select <vscale x 32 x i1> %cond, <vscale x 32 x bfloat> %splat, <vscale x 32 x bfloat> %va
+  ret <vscale x 32 x bfloat> %vc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index ee0617c9314801d9917bd22d0362819c7cc9ec82..d1049e14fa29aa551dc7414e8406c8f2d74f56d2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x i1> @llvm.vp.select.nxv1i1(<vscale x 1 x i1>, <vscale x 1 x i1>, <vscale x 1 x i1>, i32)
@@ -922,3 +922,75 @@ define <vscale x 2 x i1> @select_unknown_T_T(<vscale x 2 x i1> %x, <vscale x 2 x
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, <vscale x 2 x i1> %y, i32 %evl)
   ret <vscale x 2 x i1> %a
 }
+
+declare <vscale x 1 x bfloat> @llvm.vp.select.nxv1bf16(<vscale x 1 x i1>, <vscale x 1 x bfloat>, <vscale x 1 x bfloat>, i32)
+
+define <vscale x 1 x bfloat> @select_nxv1bf16(<vscale x 1 x i1> %a, <vscale x 1 x bfloat> %b, <vscale x 1 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x bfloat> @llvm.vp.select.nxv1bf16(<vscale x 1 x i1> %a, <vscale x 1 x bfloat> %b, <vscale x 1 x bfloat> %c, i32 %evl)
+  ret <vscale x 1 x bfloat> %v
+}
+
+declare <vscale x 2 x bfloat> @llvm.vp.select.nxv2bf16(<vscale x 2 x i1>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>, i32)
+
+define <vscale x 2 x bfloat> @select_nxv2bf16(<vscale x 2 x i1> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.select.nxv2bf16(<vscale x 2 x i1> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c, i32 %evl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+declare <vscale x 4 x bfloat> @llvm.vp.select.nxv4bf16(<vscale x 4 x i1>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>, i32)
+
+define <vscale x 4 x bfloat> @select_nxv4bf16(<vscale x 4 x i1> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x bfloat> @llvm.vp.select.nxv4bf16(<vscale x 4 x i1> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c, i32 %evl)
+  ret <vscale x 4 x bfloat> %v
+}
+
+declare <vscale x 8 x bfloat> @llvm.vp.select.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
+
+define <vscale x 8 x bfloat> @select_nxv8bf16(<vscale x 8 x i1> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x bfloat> @llvm.vp.select.nxv8bf16(<vscale x 8 x i1> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i32 %evl)
+  ret <vscale x 8 x bfloat> %v
+}
+
+declare <vscale x 16 x bfloat> @llvm.vp.select.nxv16bf16(<vscale x 16 x i1>, <vscale x 16 x bfloat>, <vscale x 16 x bfloat>, i32)
+
+define <vscale x 16 x bfloat> @select_nxv16bf16(<vscale x 16 x i1> %a, <vscale x 16 x bfloat> %b, <vscale x 16 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x bfloat> @llvm.vp.select.nxv16bf16(<vscale x 16 x i1> %a, <vscale x 16 x bfloat> %b, <vscale x 16 x bfloat> %c, i32 %evl)
+  ret <vscale x 16 x bfloat> %v
+}
+
+declare <vscale x 32 x bfloat> @llvm.vp.select.nxv32bf16(<vscale x 32 x i1>, <vscale x 32 x bfloat>, <vscale x 32 x bfloat>, i32)
+
+define <vscale x 32 x bfloat> @select_nxv32bf16(<vscale x 32 x i1> %a, <vscale x 32 x bfloat> %b, <vscale x 32 x bfloat> %c, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vp.select.nxv32bf16(<vscale x 32 x i1> %a, <vscale x 32 x bfloat> %b, <vscale x 32 x bfloat> %c, i32 %evl)
+  ret <vscale x 32 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/xcvmem.ll b/llvm/test/CodeGen/RISCV/xcvmem.ll
new file mode 100644
index 0000000000000000000000000000000000000000..037e49b9b0df7db428d229672a933325686a119d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xcvmem.ll
@@ -0,0 +1,295 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=riscv32 -mattr=+xcvmem -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i32> @lb_ri_inc(i8* %a) {
+; CHECK-LABEL: lb_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lb a1, (a0), 42
+; CHECK-NEXT:    ret
+  %1 = load i8, i8* %a
+  %2 = sext i8 %1 to i32
+  %3 = getelementptr i8, i8* %a, i32 42
+  %4 = ptrtoint i8* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define <2 x i32> @lb_rr_inc(i8* %a, i32 %b) {
+; CHECK-LABEL: lb_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lb a1, (a0), a1
+; CHECK-NEXT:    ret
+  %1 = load i8, i8* %a
+  %2 = sext i8 %1 to i32
+  %3 = getelementptr i8, i8* %a, i32 %b
+  %4 = ptrtoint i8* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define i32 @lb_rr(i8* %a, i32 %b) {
+; CHECK-LABEL: lb_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lb a0, a1(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 %b
+  %2 = load i8, i8* %1
+  %3 = sext i8 %2 to i32
+  ret i32 %3
+}
+
+define <2 x i32> @lbu_ri_inc(i8* %a) {
+; CHECK-LABEL: lbu_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lbu a1, (a0), 42
+; CHECK-NEXT:    ret
+  %1 = load i8, i8* %a
+  %2 = zext i8 %1 to i32
+  %3 = getelementptr i8, i8* %a, i32 42
+  %4 = ptrtoint i8* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define <2 x i32> @lbu_rr_inc(i8* %a, i32 %b) {
+; CHECK-LABEL: lbu_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lbu a1, (a0), a1
+; CHECK-NEXT:    ret
+  %1 = load i8, i8* %a
+  %2 = zext i8 %1 to i32
+  %3 = getelementptr i8, i8* %a, i32 %b
+  %4 = ptrtoint i8* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define i32 @lbu_rr(i8* %a, i32 %b) {
+; CHECK-LABEL: lbu_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lbu a0, a1(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 %b
+  %2 = load i8, i8* %1
+  %3 = zext i8 %2 to i32
+  ret i32 %3
+}
+
+define <2 x i32> @lh_ri_inc(i16* %a) {
+; CHECK-LABEL: lh_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lh a1, (a0), 84
+; CHECK-NEXT:    ret
+  %1 = load i16, i16* %a
+  %2 = sext i16 %1 to i32
+  %3 = getelementptr i16, i16* %a, i32 42
+  %4 = ptrtoint i16* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define <2 x i32> @lh_rr_inc(i16* %a, i32 %b) {
+; CHECK-LABEL: lh_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    cv.lh a1, (a0), a1
+; CHECK-NEXT:    ret
+  %1 = load i16, i16* %a
+  %2 = sext i16 %1 to i32
+  %3 = getelementptr i16, i16* %a, i32 %b
+  %4 = ptrtoint i16* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define i32 @lh_rr(i16* %a, i32 %b) {
+; CHECK-LABEL: lh_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    cv.lh a0, a1(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 %b
+  %2 = load i16, i16* %1
+  %3 = sext i16 %2 to i32
+  ret i32 %3
+}
+
+define <2 x i32> @lhu_ri_inc(i16* %a) {
+; CHECK-LABEL: lhu_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lhu a1, (a0), 84
+; CHECK-NEXT:    ret
+  %1 = load i16, i16* %a
+  %2 = zext i16 %1 to i32
+  %3 = getelementptr i16, i16* %a, i32 42
+  %4 = ptrtoint i16* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define <2 x i32> @lhu_rr_inc(i16* %a, i32 %b) {
+; CHECK-LABEL: lhu_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    cv.lhu a1, (a0), a1
+; CHECK-NEXT:    ret
+  %1 = load i16, i16* %a
+  %2 = zext i16 %1 to i32
+  %3 = getelementptr i16, i16* %a, i32 %b
+  %4 = ptrtoint i16* %3 to i32
+  %5 = insertelement <2 x i32> undef, i32 %4, i32 0
+  %6 = insertelement <2 x i32> %5, i32 %2, i32 1
+  ret <2 x i32> %6
+}
+
+define i32 @lhu_rr(i16* %a, i32 %b) {
+; CHECK-LABEL: lhu_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    cv.lhu a0, a1(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 %b
+  %2 = load i16, i16* %1
+  %3 = zext i16 %2 to i32
+  ret i32 %3
+}
+
+define <2 x i32> @lw_ri_inc(i32* %a) {
+; CHECK-LABEL: lw_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.lw a1, (a0), 168
+; CHECK-NEXT:    ret
+  %1 = load i32, i32* %a
+  %2 = getelementptr i32, i32* %a, i32 42
+  %3 = ptrtoint i32* %2 to i32
+  %4 = insertelement <2 x i32> undef, i32 %3, i32 0
+  %5 = insertelement <2 x i32> %4, i32 %1, i32 1
+  ret <2 x i32> %5
+}
+
+define <2 x i32> @lw_rr_inc(i32* %a, i32 %b) {
+; CHECK-LABEL: lw_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    cv.lw a1, (a0), a1
+; CHECK-NEXT:    ret
+  %1 = load i32, i32* %a
+  %2 = getelementptr i32, i32* %a, i32 %b
+  %3 = ptrtoint i32* %2 to i32
+  %4 = insertelement <2 x i32> undef, i32 %3, i32 0
+  %5 = insertelement <2 x i32> %4, i32 %1, i32 1
+  ret <2 x i32> %5
+}
+
+define i32 @lw_rr(i32* %a, i32 %b) {
+; CHECK-LABEL: lw_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    cv.lw a0, a1(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i32, i32* %a, i32 %b
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+define i8* @sb_ri_inc(i8* %a, i8 %b) {
+; CHECK-LABEL: sb_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.sb a1, (a0), 42
+; CHECK-NEXT:    ret
+  store i8 %b, i8* %a
+  %1 = getelementptr i8, i8* %a, i32 42
+  ret i8* %1
+}
+
+define i8* @sb_rr_inc(i8* %a, i8 %b, i32 %c) {
+; CHECK-LABEL: sb_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.sb a1, (a0), a2
+; CHECK-NEXT:    ret
+  store i8 %b, i8* %a
+  %1 = getelementptr i8, i8* %a, i32 %c
+  ret i8* %1
+}
+
+define void @sb_rr(i8* %a, i8 %b, i32 %c) {
+; CHECK-LABEL: sb_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.sb a1, a2(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 %c
+  store i8 %b, i8* %1
+  ret void
+}
+
+define i16* @sh_ri_inc(i16* %a, i16 %b) {
+; CHECK-LABEL: sh_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.sh a1, (a0), 84
+; CHECK-NEXT:    ret
+  store i16 %b, i16* %a
+  %1 = getelementptr i16, i16* %a, i32 42
+  ret i16* %1
+}
+
+define i16* @sh_rr_inc(i16* %a, i16 %b, i32 %c) {
+; CHECK-LABEL: sh_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a2, a2, 1
+; CHECK-NEXT:    cv.sh a1, (a0), a2
+; CHECK-NEXT:    ret
+  store i16 %b, i16* %a
+  %1 = getelementptr i16, i16* %a, i32 %c
+  ret i16* %1
+}
+
+define void @sh_rr(i16* %a, i16 %b, i32 %c) {
+; CHECK-LABEL: sh_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a2, a2, 1
+; CHECK-NEXT:    cv.sh a1, a2(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 %c
+  store i16 %b, i16* %1
+  ret void
+}
+
+define i32* @sw_ri_inc(i32* %a, i32 %b) {
+; CHECK-LABEL: sw_ri_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cv.sw a1, (a0), 168
+; CHECK-NEXT:    ret
+  store i32 %b, i32* %a
+  %1 = getelementptr i32, i32* %a, i32 42
+  ret i32* %1
+}
+
+define i32* @sw_rr_inc(i32* %a, i32 %b, i32 %c) {
+; CHECK-LABEL: sw_rr_inc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a2, a2, 2
+; CHECK-NEXT:    cv.sw a1, (a0), a2
+; CHECK-NEXT:    ret
+  store i32 %b, i32* %a
+  %1 = getelementptr i32, i32* %a, i32 %c
+  ret i32* %1
+}
+
+define void @sw_rr(i32* %a, i32 %b, i32 %c) {
+; CHECK-LABEL: sw_rr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a2, a2, 2
+; CHECK-NEXT:    cv.sw a1, a2(a0)
+; CHECK-NEXT:    ret
+  %1 = getelementptr i32, i32* %a, i32 %c
+  store i32 %b, i32* %1
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll b/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll
index d6fb70bb59a7e706869846c89bcc41d6b088c35a..ec9afc789944d5dc9b7b9ceacb0f4e62419d3ad1 100644
--- a/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll
+++ b/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll
@@ -4,16 +4,16 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK: %[[#EventTy:]] = OpTypeEvent
-; CHECK: %[[#StructEventTy:]] = OpTypeStruct %[[#EventTy]]
-; CHECK: %[[#GenPtrStructEventTy:]] = OpTypePointer Generic %[[#StructEventTy]]
-; CHECK: %[[#FunPtrStructEventTy:]] = OpTypePointer Function %[[#StructEventTy]]
-; CHECK: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy:]]
+; CHECK-DAG: %[[#EventTy:]] = OpTypeEvent
+; CHECK-DAG: %[[#StructEventTy:]] = OpTypeStruct %[[#EventTy]]
+; CHECK-DAG: %[[#FunPtrStructEventTy:]] = OpTypePointer Function %[[#StructEventTy]]
+; CHECK-DAG: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy]]
+; CHECK-DAG: %[[#FunPtrEventTy:]] = OpTypePointer Function %[[#EventTy]]
 ; CHECK: OpFunction
 ; CHECK: %[[#Var:]] = OpVariable %[[#FunPtrStructEventTy]] Function
-; CHECK-NEXT: %[[#AddrspacecastVar:]] = OpPtrCastToGeneric %[[#GenPtrStructEventTy]] %[[#Var]]
-; CHECK-NEXT: %[[#BitcastVar:]] = OpBitcast %[[#GenPtrEventTy]] %[[#AddrspacecastVar]]
-; CHECK-NEXT: OpGroupWaitEvents %[[#]] %[[#]] %[[#BitcastVar]]
+; CHECK-NEXT: %[[#FunEvent:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]]
+; CHECK-NEXT: %[[#GenEvent:]] = OpPtrCastToGeneric %[[#GenPtrEventTy]] %[[#FunEvent]]
+; CHECK-NEXT: OpGroupWaitEvents %[[#]] %[[#]] %[[#GenEvent]]
 
 %"class.sycl::_V1::device_event" = type { target("spirv.Event") }
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
index b5df462bd8fa9b4795802f228d1f9c4af319ff8f..986d88da418328298555b4ca6106fc02ced2b947 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
@@ -1,9 +1,9 @@
 ; Modified from: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_usm_storage_classes/intel_usm_addrspaces.ll
 
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_usm_storage_classes %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-EXT
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_usm_storage_classes %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_usm_storage_classes %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-WITHOUT
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-: Capability USMStorageClassesINTEL
 ; CHECK-SPIRV-WITHOUT-NO: Capability USMStorageClassesINTEL
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
index 63aade4f7f8daf6a7523b22e55c2b348b7afa440..a38c9072ed1bd44d93c0fe1427bd041125f38892 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
@@ -17,7 +17,7 @@
 ; CHECK-DAG: %[[ScopeSubgroup:.*]] = OpConstant %[[TyInt32]] 3
 ; CHECK-DAG: %[[ConstInt2:.*]] = OpConstant %[[TyInt32]] 2
 ; CHECK-DAG: %[[ConstInt4:.*]] = OpConstant %[[TyInt32]] 4
-		
+
 target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
 target triple = "spir"
 
diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll
index f728eda079860d4c9937f38f8b421f8d9b9f2822..0e2b26c83bcddada4b0da8663cffe9b7c2e958f6 100644
--- a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll
+++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -print-after-all -o - 2>&1 | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -print-after-all -o - 2>&1 | FileCheck %s
 
 ; CHECK: *** IR Dump After SPIRV emit intrinsics (emit-intrinsics) ***
 
 define spir_kernel void @test_pointer_cast(ptr addrspace(1) %src) {
-; CHECK-NOT: call void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i8 undef, i32 1)
-; CHECK: call void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i32 0, i32 1)
+; CHECK-NOT: call{{.*}} void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i8 undef, i32 1)
+; CHECK: call{{.*}} void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i32 poison, i32 1)
   %b = bitcast ptr addrspace(1) %src to ptr addrspace(1)
   %g = getelementptr inbounds i32, ptr addrspace(1) %b, i64 52
   ret void
diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll
index 7056b9cb1230dcb9f3438d220264b39eb6f0d060..a9f169a5977a9a9dc2008845ed1f4ba2f2037a98 100644
--- a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll
+++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll
@@ -1,11 +1,12 @@
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -print-after-all -o - 2>&1 | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -print-after-all -o - 2>&1 | FileCheck %s
 
 ; CHECK: *** IR Dump After SPIRV emit intrinsics (emit-intrinsics) ***
 
 define spir_kernel void @test(ptr addrspace(1) %srcimg) {
-; CHECK: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) undef)
+; CHECK: call{{.*}} void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison)
   %call1 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg)
-; CHECK-NOT: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) undef)
+; CHECK-NOT: call{{.*}} void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison)
   %call2 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg)
   ret void
 ; CHECK: }
diff --git a/llvm/test/CodeGen/SPIRV/pointers/duplicate-type-ptr-def.ll b/llvm/test/CodeGen/SPIRV/pointers/duplicate-type-ptr-def.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8e70bef3a5399a9e08aff5f4f28a6b5486347dd2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/duplicate-type-ptr-def.ll
@@ -0,0 +1,26 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: %[[#Char:]] = OpTypeInt 8 0
+; CHECK: %[[#Ptr:]] = OpTypePointer Generic %[[#Char]]
+; CHECK: %[[#TypeDef:]] = OpTypePointer Generic %[[#Ptr]]
+; CHECK-NOT: %[[#TypeDef]] = OpTypePointer Generic %[[#Ptr]]
+
+%Range = type { %Array }
+%Array = type { [1 x i64] }
+
+define spir_func ptr addrspace(4) @foo(ptr addrspace(4) dereferenceable_or_null(32) %this) {
+entry:
+  %addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %addr
+  %arrayidx = getelementptr inbounds ptr addrspace(4), ptr addrspace(1) null, i64 0
+  %r = addrspacecast ptr addrspace(1) %arrayidx to ptr addrspace(4)
+  ret ptr addrspace(4) %r
+}
+
+define spir_func void @bar() {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-dup.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-dup.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e6130cedf68000147cc36a5198393bc44d2eb784
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-dup.ll
@@ -0,0 +1,27 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV-DAG: %[[#Char:]] = OpTypeInt 8 0
+; CHECK-SPIRV-DAG: %[[#ArrayTy:]] = OpTypeArray %[[#Char]] %[[#]]
+; CHECK-SPIRV-DAG: %[[#CharPtrTy:]] = OpTypePointer CrossWorkgroup %[[#Char]]
+; CHECK-SPIRV-DAG: %[[#Const1:]] = OpConstantComposite %[[#]] %[[#]] %[[#]]
+; CHECK-SPIRV-DAG: %[[#CharPtrPtrTy:]] = OpTypePointer CrossWorkgroup %[[#CharPtrTy]]
+; CHECK-SPIRV-DAG: %[[#PtrArrayTy:]] = OpTypePointer CrossWorkgroup %[[#ArrayTy]]
+; CHECK-SPIRV-DAG: OpVariable %[[#PtrArrayTy]] CrossWorkgroup %[[#Const1]]
+; CHECK-SPIRV-DAG: OpVariable %[[#CharPtrPtrTy]] CrossWorkgroup %[[#]]
+
+@a_var = addrspace(1) global [2 x i8] c"\01\01"
+@p_var = addrspace(1) global ptr addrspace(1) getelementptr inbounds ([2 x i8], ptr addrspace(1) @a_var, i32 0, i64 1)
+
+define spir_func zeroext i8 @foo() {
+entry:
+  ret i8 1
+}
+
+define spir_func zeroext i8 @bar() {
+entry:
+  ret i8 1
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-simple-for.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-simple-for.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ab7f923797f309d90206ed64916bd773d13f92db
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-simple-for.ll
@@ -0,0 +1,65 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - --translator-compatibility-mode | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-COMPAT,CHECK-COMPAT64
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-DEFVER
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - --translator-compatibility-mode | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-COMPAT,CHECK-COMPAT32
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV-DAG: %[[#CharTy:]] = OpTypeInt 8 0
+; CHECK-SPIRV-DAG: %[[#IntTy:]] = OpTypeInt 32 0
+; CHECK-SPIRV-DAG: %[[#IntConst1:]] = OpConstant %[[#IntTy]] 1
+; CHECK-SPIRV-DAG: %[[#ArrTy:]] = OpTypeArray %[[#IntTy]] %[[#IntConst1]]
+; CHECK-SPIRV-DAG: %[[#BoolTy:]] = OpTypeBool
+; CHECK-SPIRV-DAG: %[[#LongTy:]] = OpTypeInt 64 0
+; CHECK-SPIRV-DAG: %[[#LongConst4:]] = OpConstant %[[#LongTy]] 4
+; CHECK-SPIRV-DAG: %[[#IntConst123:]] = OpConstant %[[#IntTy]] 123
+; CHECK-SPIRV-DAG: %[[#CharPtrTy:]] = OpTypePointer Function %[[#CharTy]]
+; CHECK-SPIRV-DAG: %[[#ArrPtrTy:]] = OpTypePointer Function %[[#ArrTy]]
+; CHECK-SPIRV-DAG: %[[#IntPtrTy:]] = OpTypePointer Function %[[#IntTy]]
+; CHECK-SPIRV: OpFunction
+; CHECK-SPIRV: %[[#LblEntry:]] = OpLabel
+; CHECK-SPIRV: %[[#Value:]] = OpVariable %[[#ArrPtrTy]] Function
+; CHECK-SPIRV: %[[#ValueAsCharPtr:]] = OpBitcast %[[#CharPtrTy]] %[[#Value]]
+; CHECK-SPIRV: %[[#Eof:]] = OpInBoundsPtrAccessChain %[[#CharPtrTy]] %[[#ValueAsCharPtr]] %[[#LongConst4]]
+; CHECK-SPIRV: %[[#EofAsArray:]] = OpBitcast %[[#ArrPtrTy]] %[[#Eof]]
+; CHECK-SPIRV: OpBranch %[[#LblCond:]]
+; CHECK-SPIRV: %[[#LblCond]] = OpLabel
+; CHECK-SPIRV: %[[#Iter:]] = OpPhi %[[#ArrPtrTy]] %[[#Value]] %[[#LblEntry]] %[[#CurrValue:]] %[[#LblBody:]]
+; CHECK-COMPAT64: %[[#IterInt:]] = OpConvertPtrToU %[[#LongTy]] %[[#Iter]]
+; CHECK-COMPAT64: %[[#EofInt:]] = OpConvertPtrToU %[[#LongTy]] %[[#EofAsArray]]
+; CHECK-COMPAT32: %[[#IterInt:]] = OpConvertPtrToU %[[#IntTy]] %[[#Iter]]
+; CHECK-COMPAT32: %[[#EofInt:]] = OpConvertPtrToU %[[#IntTy]] %[[#EofAsArray]]
+; CHECK-COMPAT: %[[#Is:]] = OpIEqual %[[#BoolTy]] %[[#IterInt]] %[[#EofInt]]
+; CHECK-DEFVER: %[[#Is:]] = OpPtrEqual %[[#BoolTy]] %[[#Iter]] %[[#EofAsArray]]
+; CHECK-SPIRV: OpBranchConditional %[[#Is]] %[[#LblExit:]] %[[#LblBody]]
+; CHECK-SPIRV: %[[#LblBody]] = OpLabel
+; CHECK-SPIRV: %[[#IterAsIntPtr:]] = OpBitcast %[[#IntPtrTy]] %[[#Iter]]
+; CHECK-SPIRV: OpStore %[[#IterAsIntPtr]] %[[#IntConst123]] Aligned 4
+; CHECK-SPIRV: %[[#IterAsCharPtr:]] = OpBitcast %[[#CharPtrTy]] %[[#Iter]]
+; CHECK-SPIRV: %[[#CurrValueAsCharPtr:]] = OpInBoundsPtrAccessChain %[[#CharPtrTy]] %[[#IterAsCharPtr]] %[[#LongConst4]]
+; CHECK-SPIRV: %[[#CurrValue]] = OpBitcast %[[#ArrPtrTy]] %[[#CurrValueAsCharPtr]]
+; CHECK-SPIRV: OpBranch %[[#LblCond]]
+; CHECK-SPIRV: %[[#LblExit]] = OpLabel
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_kernel void @foo() {
+entry:
+  %v = alloca [1 x i32], align 4
+  %eof = getelementptr inbounds i8, ptr %v, i64 4
+  br label %cond
+
+cond:
+  %iter = phi ptr [ %v, %entry ], [ %curr, %body ]
+  %is = icmp eq ptr %iter, %eof
+  br i1 %is, label %exit, label %body
+
+body:
+  store i32 123, ptr %iter, align 4
+  %curr = getelementptr inbounds i8, ptr %iter, i64 4
+  br label %cond
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll
new file mode 100644
index 0000000000000000000000000000000000000000..96d6016083f06603f54815118221beda21e62f50
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll
@@ -0,0 +1,36 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV-DAG: %[[#LongTy:]] = OpTypeInt 64 0
+; CHECK-SPIRV-DAG: %[[#IntTy:]] = OpTypeInt 32 0
+; CHECK-SPIRV-DAG: %[[#EventTy:]] = OpTypeEvent
+; CHECK-SPIRV-DAG: %[[#Scope:]] = OpConstant %[[#IntTy]] 2
+; CHECK-SPIRV-DAG: %[[#Num:]] = OpConstant %[[#LongTy]] 123
+; CHECK-SPIRV-DAG: %[[#Null:]] = OpConstantNull
+; CHECK-SPIRV-DAG: %[[#Stride:]] = OpConstant %[[#LongTy]] 1
+; CHECK-SPIRV-DAG: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy]]
+; CHECK-SPIRV-DAG: %[[#FunPtrEventTy:]] = OpTypePointer Function %[[#EventTy]]
+; CHECK-SPIRV: OpFunction
+; CHECK-SPIRV: %[[#Var:]] = OpVariable %[[#]] Function
+; CHECK-SPIRV: %[[#ResEvent:]] = OpGroupAsyncCopy %[[#EventTy]] %[[#Scope]] %[[#Null]] %[[#Null]] %[[#Num]] %[[#Stride]] %[[#Null]]
+; CHECK-SPIRV: %[[#VarPtrEvent:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]]
+; CHECK-SPIRV: OpStore %[[#VarPtrEvent]] %[[#ResEvent]]
+; CHECK-SPIRV: %[[#VarPtrEvent2:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]]
+; CHECK-SPIRV: %[[#PtrEventGen:]] = OpPtrCastToGeneric %[[#]] %[[#VarPtrEvent2]]
+; CHECK-SPIRV: OpGroupWaitEvents %[[#Scope]] %[[#Num]] %[[#PtrEventGen]]
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_kernel void @foo() {
+  %event = alloca ptr, align 8
+  %call = call spir_func ptr @_Z29async_work_group_strided_copyPU3AS3hPU3AS1Khmm9ocl_event(ptr null, ptr null, i64 123, i64 1, ptr null)
+  store ptr %call, ptr %event, align 8
+  %event.ascast = addrspacecast ptr %event to ptr addrspace(4)
+  call spir_func void @_Z17wait_group_eventsiPU3AS49ocl_event(i64 123, ptr addrspace(4) %event.ascast)
+  ret void
+}
+
+declare spir_func ptr @_Z29async_work_group_strided_copyPU3AS3hPU3AS1Khmm9ocl_event(ptr, ptr, i64, i64, ptr)
+declare spir_func void @_Z17wait_group_eventsiPU3AS49ocl_event(i64, ptr addrspace(4))
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/bitcast-diff-addrspace.ll b/llvm/test/CodeGen/SPIRV/transcoding/bitcast-diff-addrspace.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1ef3041c4ce567d6d69d9ebdd1ccd9ac66433c8
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/transcoding/bitcast-diff-addrspace.ll
@@ -0,0 +1,44 @@
+; Check that bitcast for pointers implies that the address spaces must match.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_INTEL_usm_storage_classes -o - | FileCheck %s
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s --spirv-ext=+SPV_INTEL_usm_storage_classes -o - -filetype=obj | spirv-val %}
+
+; CHECK: Capability USMStorageClassesINTEL
+; CHECK: OpExtension "SPV_INTEL_usm_storage_classes"
+; CHECK-DAG: OpName %[[#Bar:]] "bar"
+; CHECK-DAG: %[[#Void:]] = OpTypeVoid
+; CHECK-DAG: %[[#Char:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#GenPtrChar:]] = OpTypePointer Generic %[[#Char]]
+; CHECK-DAG: %[[#AtDevice:]] = OpTypePointer DeviceOnlyINTEL %[[#Char]]
+; CHECK-DAG: %[[#AtHost:]] = OpTypePointer HostOnlyINTEL %[[#Char]]
+; CHECK-DAG: %[[#PtrVarHost:]] = OpTypePointer Function %[[#AtHost]]
+; CHECK-DAG: %[[#PtrVarDevice:]] = OpTypePointer Function %[[#AtDevice]]
+; CHECK: OpFunction
+; CHECK: %[[#VarDevice:]] = OpVariable %[[#PtrVarDevice]] Function
+; CHECK: %[[#VarHost:]] = OpVariable %[[#PtrVarHost]] Function
+; CHECK: %[[#LoadedDevice:]] = OpLoad %[[#AtDevice]] %[[#VarDevice]]
+; CHECK: %[[#CastedFromDevice:]] = OpPtrCastToGeneric %[[#GenPtrChar]] %[[#LoadedDevice]]
+; CHECK: OpFunctionCall %[[#Void]] %[[#Bar]] %[[#CastedFromDevice]]
+; CHECK: %[[#LoadedHost:]] = OpLoad %[[#AtHost]] %[[#VarHost]]
+; CHECK: %[[#CastedFromHost:]] = OpPtrCastToGeneric %[[#GenPtrChar]] %[[#LoadedHost]]
+; CHECK: OpFunctionCall %[[#Void]] %[[#Bar]] %[[#CastedFromHost]]
+
+define spir_func void @foo() {
+entry:
+  %device_var = alloca ptr addrspace(5)
+  %host_var = alloca ptr addrspace(6)
+  %p1 = load ptr addrspace(5), ptr %device_var
+  %p2 = addrspacecast ptr addrspace(5) %p1 to ptr addrspace(4)
+  call spir_func void @bar(ptr addrspace(4) %p2)
+  %p3 = load ptr addrspace(6), ptr %host_var
+  %p4 = addrspacecast ptr addrspace(6) %p3 to ptr addrspace(4)
+  call spir_func void @bar(ptr addrspace(4) %p4)
+  ret void
+}
+
+define spir_func void @bar(ptr addrspace(4) %data) {
+entry:
+  %data.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %data, ptr %data.addr
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll
index d0c4dff43121ccfba0b54a13adc94802fb09df2e..53883fd1691f5e443881a9e84f78aa61abdf4b63 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll
@@ -32,9 +32,9 @@
 ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] Constant
 ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] Constant
 ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] Constant
-; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] LinkageAttributes "__spirv_BuiltInGlobalOffset" Import 
-; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] LinkageAttributes "__spirv_BuiltInGlobalSize" Import 
-; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] LinkageAttributes "__spirv_BuiltInGlobalInvocationId" Import 
+; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] LinkageAttributes "__spirv_BuiltInGlobalOffset" Import
+; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] LinkageAttributes "__spirv_BuiltInGlobalSize" Import
+; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] LinkageAttributes "__spirv_BuiltInGlobalInvocationId" Import
 
 %"class._ZTSN2cl4sycl5rangeILi2EEE.cl::sycl::range" = type { %"class._ZTSN2cl4sycl6detail5arrayILi2EEE.cl::sycl::detail::array" }
 %"class._ZTSN2cl4sycl6detail5arrayILi2EEE.cl::sycl::detail::array" = type { [2 x i64] }
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll
index 824ca1b2d692496d01c8db25ebe4b1a5506ab266..6f61aba23a46fa69fd712a476c56494729614b6f 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV: %[[#IMAGE_TYPE:]] = OpTypeImage
 ; CHECK-SPIRV: %[[#IMAGE_ARG:]] = OpFunctionParameter %[[#IMAGE_TYPE]]
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll b/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll
index 9654de6b84132c6fe2d3ea9126d2a9fc88c9ec84..1073473a224df790a6d2cba7f70d73e95ea79021 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll
@@ -5,7 +5,7 @@
 ;; }
 ;;
 ;; kernel void testSubGroupNonUniformAll(global int* dst) {
-;; 	dst[0] = sub_group_non_uniform_all(0); 
+;; 	dst[0] = sub_group_non_uniform_all(0);
 ;; }
 ;;
 ;; kernel void testSubGroupNonUniformAny(global int* dst) {
diff --git a/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir b/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir
index bcd7f51cdf7802c0c5c24c431dabacd2f7384d71..917cc776e3ef094775ee449426ce4b2cde51305b 100644
--- a/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir
+++ b/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir
@@ -1,4 +1,5 @@
 # RUN: llc -verify-machineinstrs -run-pass regallocfast -mtriple s390x-ibm-linux -o - %s | FileCheck %s
+# RUN: llc -verify-machineinstrs -passes=regallocfast -mtriple s390x-ibm-linux -o - %s | FileCheck %s
 --- |
   
   @g_167 = external global [5 x i64], align 8
diff --git a/llvm/test/CodeGen/Thumb/frame-access.ll b/llvm/test/CodeGen/Thumb/frame-access.ll
index 422c595472ee490d7cf292d73de51ca2d798995c..07a6b6d0b94906d468147736feb1163fc67b80b2 100644
--- a/llvm/test/CodeGen/Thumb/frame-access.ll
+++ b/llvm/test/CodeGen/Thumb/frame-access.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-ATPCS
 ; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-ATPCS,CHECK-ATPCS
-; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none -mattr=+aapcs-frame-chain-leaf %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-AAPCS
-; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all -mattr=+aapcs-frame-chain-leaf %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-AAPCS,CHECK-AAPCS
+; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none -mattr=+aapcs-frame-chain %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-AAPCS
+; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all -mattr=+aapcs-frame-chain %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-AAPCS,CHECK-AAPCS
 
 ; struct S { int x[128]; } s;
 ; int f(int *, int, int, int, struct S);
diff --git a/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll b/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll
index 37dd16bd9dd92d55928af1cf07ed766ccaf1b73e..d62eb20b84593996720ddbd20d10e751e5aad281 100644
--- a/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll
+++ b/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll
@@ -1,27 +1,24 @@
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=RESERVED-R7
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
-; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all                                2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all      -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE     --check-prefix=R11-RESERVED
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf                           2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE     --check-prefix=R11-RESERVED
+; RUN:     llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none                               2>&1 | FileCheck %s --check-prefix=R7-FREE     --check-prefix=R11-FREE
+; RUN:     llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none     -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE     --check-prefix=R11-FREE
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved                           2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE
+; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE     --check-prefix=R11-RESERVED
 
 declare void @leaf(i32 %input)
 
 define void @reserved_r7(i32 %input) {
-; RESERVED-NONE-NOT: error: write to reserved register 'R7'
-; RESERVED-R7: error: write to reserved register 'R7'
-; RESERVED-R11-NOT: error: write to reserved register 'R7'
+; R7-RESERVED: error: write to reserved register 'R7'
+; R7-FREE-NOT: error: write to reserved register 'R7'
   %1 = call i32 asm sideeffect "mov $0, $1", "={r7},r"(i32 %input)
   ret void
 }
 
 define void @reserved_r11(i32 %input) {
-; RESERVED-NONE-NOT: error: write to reserved register 'R11'
-; RESERVED-R7-NOT: error: write to reserved register 'R11'
-; RESERVED-R11: error: write to reserved register 'R11'
+; R11-RESERVED: error: write to reserved register 'R11'
+; R11-FREE-NOT: error: write to reserved register 'R11'
   %1 = call i32 asm sideeffect "mov $0, $1", "={r11},r"(i32 %input)
   ret void
 }
diff --git a/llvm/test/CodeGen/Thumb/frame-chain.ll b/llvm/test/CodeGen/Thumb/frame-chain.ll
index c92235e0f8279ede52901e4427feb0149066b537..eb62ce09caf1be45c20ad0c2b23b9d221739a5d0 100644
--- a/llvm/test/CodeGen/Thumb/frame-chain.ll
+++ b/llvm/test/CodeGen/Thumb/frame-chain.ll
@@ -1,12 +1,9 @@
 ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all --verify-machineinstrs | FileCheck %s --check-prefixes=FP,LEAF-FP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS
+; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS
 ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP,LEAF-NOFP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS
+; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS
 ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP
-; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS
+; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS
 
 define dso_local noundef i32 @leaf(i32 noundef %0) {
 ; LEAF-FP-LABEL: leaf:
diff --git a/llvm/test/CodeGen/Thumb/high-reg-clobber.mir b/llvm/test/CodeGen/Thumb/high-reg-clobber.mir
index ab73f1448d715b80a2675e4e9ba13d721ce9b9e3..1402c7c2cbca36d053eec8726368307f21065d2e 100644
--- a/llvm/test/CodeGen/Thumb/high-reg-clobber.mir
+++ b/llvm/test/CodeGen/Thumb/high-reg-clobber.mir
@@ -2,6 +2,7 @@
 # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass regallocbasic %s -o - | FileCheck %s
 # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass greedy %s -o - | FileCheck %s
 # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass regallocfast %s -o - | FileCheck %s --check-prefix=FAST
+# RUN: llc -mtriple thumbv6m-arm-none-eabi -passes=regallocfast %s -o - | FileCheck %s --check-prefix=FAST
 
 ...
 ---
diff --git a/llvm/test/CodeGen/Thumb2/high-reg-spill.mir b/llvm/test/CodeGen/Thumb2/high-reg-spill.mir
index 1cf58da63fe831e2e32dfb6b1551702a91faf484..8c92020fb0519da0097d3b650fffea805359aa09 100644
--- a/llvm/test/CodeGen/Thumb2/high-reg-spill.mir
+++ b/llvm/test/CodeGen/Thumb2/high-reg-spill.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -run-pass regallocfast %s -o - | FileCheck %s
+# RUN: llc -passes=regallocfast %s -o - | FileCheck %s
 
 # This test examines register allocation and spilling with Fast Register
 # Allocator. The test uses inline assembler that requests an input variable to
diff --git a/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll b/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll
index f9b3757bb6d2ce80d281eb3298646172ea24173d..6392452e06cddde259b3ae9d1a75f96a9fff9fc9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll
@@ -20,50 +20,35 @@ define void @convert_vptblock(ptr %pchTarget, i16 signext %iTargetStride, ptr %p
 ; CHECK-NEXT:    mov.w r8, #0
 ; CHECK-NEXT:    ldrd r4, r5, [sp, #88]
 ; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    cmp.w r10, #8
-; CHECK-NEXT:    mov.w r0, #1
-; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    mov.w r11, #0
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge r3, #8
 ; CHECK-NEXT:    vidup.u16 q0, r8, #4
-; CHECK-NEXT:    sub.w r3, r10, r3
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    adds r3, #7
 ; CHECK-NEXT:    vmov.i16 q2, #0x100
 ; CHECK-NEXT:    vmov.i16 q3, #0xff
-; CHECK-NEXT:    add.w r9, r0, r3, lsr #3
 ; CHECK-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB0_3 Depth 2
-; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r6, r8
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    dls lr, r9
+; CHECK-NEXT:    dlstp.16 lr, r10
 ; CHECK-NEXT:  .LBB0_3: @ %do.body
 ; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vctp.16 r3
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u16 q5, [r2, q4]
+; CHECK-NEXT:    vldrb.u16 q5, [r2, q4]
 ; CHECK-NEXT:    vmul.i16 q4, q5, r5
 ; CHECK-NEXT:    vshr.u16 q4, q4, #8
 ; CHECK-NEXT:    vsub.i16 q5, q2, q4
 ; CHECK-NEXT:    vpt.i16 eq, q4, q3
 ; CHECK-NEXT:    vmovt q5, q1
-; CHECK-NEXT:    vctp.16 r3
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u16 q6, [r0]
+; CHECK-NEXT:    vldrb.u16 q6, [r0]
 ; CHECK-NEXT:    vsub.i16 q4, q2, q5
-; CHECK-NEXT:    subs r3, #8
 ; CHECK-NEXT:    vmul.i16 q5, q5, q6
 ; CHECK-NEXT:    vmla.i16 q5, q4, r4
 ; CHECK-NEXT:    vshr.u16 q4, q5, #8
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.16 q4, [r0], #8
+; CHECK-NEXT:    vstrb.16 q4, [r0], #8
 ; CHECK-NEXT:    vidup.u16 q4, r6, #4
-; CHECK-NEXT:    le lr, .LBB0_3
+; CHECK-NEXT:    letp lr, .LBB0_3
 ; CHECK-NEXT:  @ %bb.4: @ %do.end
 ; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    add.w r0, r11, #1
diff --git a/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll
new file mode 100644
index 0000000000000000000000000000000000000000..80f3db0e52e0c76dec53ddf3ac33ca9436d17def
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll
@@ -0,0 +1,484 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
+; RUN: opt -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s
+; REQUIRES: webassembly-registered-target
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+; Check the variables are lowered to the locations this target expects
+
+; The types show the call frames
+; CHECK: %single_i32.vararg = type <{ i32 }>
+; CHECK: %single_double.vararg = type <{ double }>
+; CHECK: %single_v4f32.vararg = type <{ <4 x float> }>
+; CHECK: %single_v8f32.vararg = type <{ <8 x float> }>
+; CHECK: %single_v16f32.vararg = type <{ <16 x float> }>
+; CHECK: %single_v32f32.vararg = type <{ <32 x float> }>
+; CHECK: %i32_double.vararg = type <{ i32, [4 x i8], double }>
+; CHECK: %double_i32.vararg = type <{ double, i32 }>
+; CHECK: %i32_libcS.vararg = type <{ i32, ptr }>
+; CHECK: %libcS_i32.vararg = type <{ ptr, i32 }>
+; CHECK: %i32_v4f32.vararg = type <{ i32, [12 x i8], <4 x float> }>
+; CHECK: %v4f32_i32.vararg = type <{ <4 x float>, i32 }>
+; CHECK: %i32_v8f32.vararg = type <{ i32, [28 x i8], <8 x float> }>
+; CHECK: %v8f32_i32.vararg = type <{ <8 x float>, i32 }>
+; CHECK: %i32_v16f32.vararg = type <{ i32, [60 x i8], <16 x float> }>
+; CHECK: %v16f32_i32.vararg = type <{ <16 x float>, i32 }>
+; CHECK: %i32_v32f32.vararg = type <{ i32, [124 x i8], <32 x float> }>
+; CHECK: %v32f32_i32.vararg = type <{ <32 x float>, i32 }>
+; CHECK: %fptr_single_i32.vararg = type <{ i32 }>
+; CHECK: %fptr_libcS.vararg = type <{ ptr }>
+
+%struct.libcS = type { i8, i16, i32, i32, float, double }
+
+@vararg_ptr = hidden global ptr @vararg, align 4
+
+define hidden void @copy(ptr noundef %va) {
+; CHECK-LABEL: define {{[^@]+}}@copy(ptr noundef %va) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %va.addr = alloca ptr, align 4
+; CHECK-NEXT:    %cp = alloca ptr, align 4
+; CHECK-NEXT:    store ptr %va, ptr %va.addr, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr %cp, ptr %va.addr, i32 4, i1 false)
+; CHECK-NEXT:    %0 = load ptr, ptr %cp, align 4
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %va.addr = alloca ptr, align 4
+  %cp = alloca ptr, align 4
+  store ptr %va, ptr %va.addr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr)
+  %0 = load ptr, ptr %cp, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+
+declare void @llvm.va_copy.p0(ptr, ptr)
+
+declare void @valist(ptr noundef)
+
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+define hidden void @start_once(...) {
+; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %s = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+; CHECK-NEXT:    store ptr %varargs, ptr %s, align 4
+; CHECK-NEXT:    %0 = load ptr, ptr %s, align 4
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %s = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+  call void @llvm.va_start.p0(ptr nonnull %s)
+  %0 = load ptr, ptr %s, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+  ret void
+}
+
+declare void @llvm.va_start.p0(ptr)
+
+declare void @llvm.va_end.p0(ptr)
+
+define hidden void @start_twice(...) {
+; CHECK-LABEL: define {{[^@]+}}@start_twice(ptr %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %s0 = alloca ptr, align 4
+; CHECK-NEXT:    %s1 = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+; CHECK-NEXT:    store ptr %varargs, ptr %s0, align 4
+; CHECK-NEXT:    %0 = load ptr, ptr %s0, align 4
+; CHECK-NEXT:    call void @valist(ptr noundef %0)
+; CHECK-NEXT:    store ptr %varargs, ptr %s1, align 4
+; CHECK-NEXT:    %1 = load ptr, ptr %s1, align 4
+; CHECK-NEXT:    call void @valist(ptr noundef %1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %s0 = alloca ptr, align 4
+  %s1 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.va_start.p0(ptr nonnull %s0)
+  %0 = load ptr, ptr %s0, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s0)
+  call void @llvm.va_start.p0(ptr nonnull %s1)
+  %1 = load ptr, ptr %s1, align 4
+  call void @valist(ptr noundef %1)
+  call void @llvm.va_end.p0(ptr %s1)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+  ret void
+}
+
+define hidden void @single_i32(i32 noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_i32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x)
+  ret void
+}
+
+declare void @vararg(...)
+
+define hidden void @single_double(double noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_double.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_double.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store double %x, ptr %0, align 8
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(double noundef %x)
+  ret void
+}
+
+define hidden void @single_v4f32(<4 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v4f32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <4 x float> %x, ptr %0, align 16
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<4 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v8f32(<8 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v8f32.vararg, align 32
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <8 x float> %x, ptr %0, align 32
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<8 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v16f32(<16 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v16f32.vararg, align 64
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <16 x float> %x, ptr %0, align 64
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<16 x float> noundef %x)
+  ret void
+}
+
+define hidden void @single_v32f32(<32 x float> noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %single_v32f32.vararg, align 128
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %single_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <32 x float> %x, ptr %0, align 128
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<32 x float> noundef %x)
+  ret void
+}
+
+define hidden void @i32_double(i32 noundef %x, double noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_double.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 2
+; CHECK-NEXT:    store double %y, ptr %1, align 8
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, double noundef %y)
+  ret void
+}
+
+define hidden void @double_i32(double noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %double_i32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store double %x, ptr %0, align 8
+; CHECK-NEXT:    %1 = getelementptr inbounds %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(double noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_libcS(i32 noundef %x, ptr noundef byval(%struct.libcS) align 8 %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_libcS(i32 noundef %x, ptr noundef byval(%struct.libcS) align 8 %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %IndirectAlloca = alloca %struct.libcS, align 8
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_libcS.vararg, align 16
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %y, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, ptr noundef nonnull byval(%struct.libcS) align 8 %y)
+  ret void
+}
+
+define hidden void @libcS_i32(ptr noundef byval(%struct.libcS) align 8 %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@libcS_i32(ptr noundef byval(%struct.libcS) align 8 %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %IndirectAlloca = alloca %struct.libcS, align 8
+; CHECK-NEXT:    %vararg_buffer = alloca %libcS_i32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(ptr noundef nonnull byval(%struct.libcS) align 8 %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v4f32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 2
+; CHECK-NEXT:    store <4 x float> %y, ptr %1, align 16
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <4 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v4f32_i32.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <4 x float> %x, ptr %0, align 16
+; CHECK-NEXT:    %1 = getelementptr inbounds %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<4 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v8f32.vararg, align 32
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 2
+; CHECK-NEXT:    store <8 x float> %y, ptr %1, align 32
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <8 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v8f32_i32.vararg, align 32
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 36, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <8 x float> %x, ptr %0, align 32
+; CHECK-NEXT:    %1 = getelementptr inbounds %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 36, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<8 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v16f32.vararg, align 64
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 2
+; CHECK-NEXT:    store <16 x float> %y, ptr %1, align 64
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <16 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v16f32_i32.vararg, align 64
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 68, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <16 x float> %x, ptr %0, align 64
+; CHECK-NEXT:    %1 = getelementptr inbounds %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 68, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<16 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %i32_v32f32.vararg, align 128
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 2
+; CHECK-NEXT:    store <32 x float> %y, ptr %1, align 128
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(i32 noundef %x, <32 x float> noundef %y)
+  ret void
+}
+
+define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
+; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %v32f32_i32.vararg, align 128
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 132, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store <32 x float> %x, ptr %0, align 128
+; CHECK-NEXT:    %1 = getelementptr inbounds %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store i32 %y, ptr %1, align 4
+; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 132, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  tail call void (...) @vararg(<32 x float> noundef %x, i32 noundef %y)
+  ret void
+}
+
+define hidden void @fptr_single_i32(i32 noundef %x) {
+; CHECK-LABEL: define {{[^@]+}}@fptr_single_i32(i32 noundef %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %fptr_single_i32.vararg, align 16
+; CHECK-NEXT:    %0 = load volatile ptr, ptr @vararg_ptr, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    %1 = getelementptr inbounds %fptr_single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %1, align 4
+; CHECK-NEXT:    call void %0(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr @vararg_ptr, align 4
+  tail call void (...) %0(i32 noundef %x)
+  ret void
+}
+
+define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) {
+; CHECK-LABEL: define {{[^@]+}}@fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %IndirectAlloca = alloca %struct.libcS, align 8
+; CHECK-NEXT:    %vararg_buffer = alloca %fptr_libcS.vararg, align 16
+; CHECK-NEXT:    %0 = load volatile ptr, ptr @vararg_ptr, align 4
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    %1 = getelementptr inbounds %fptr_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %1, align 4
+; CHECK-NEXT:    call void %0(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr @vararg_ptr, align 4
+  tail call void (...) %0(ptr noundef nonnull byval(%struct.libcS) align 8 %x)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index 761a75418a00f84ae752f6bc33832fe31a836694..67388b688e3bb7efa8ccee497229f8064848df1a 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -11788,27 +11788,35 @@ define <4 x float> @minnum_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) {
 ; NO-SIMD128-LABEL: minnum_intrinsic_v4f32:
 ; NO-SIMD128:         .functype minnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    call $push0=, fminf, $4, $8
-; NO-SIMD128-NEXT:    f32.store 12($0), $pop0
-; NO-SIMD128-NEXT:    call $push1=, fminf, $3, $7
-; NO-SIMD128-NEXT:    f32.store 8($0), $pop1
-; NO-SIMD128-NEXT:    call $push2=, fminf, $2, $6
-; NO-SIMD128-NEXT:    f32.store 4($0), $pop2
-; NO-SIMD128-NEXT:    call $push3=, fminf, $1, $5
-; NO-SIMD128-NEXT:    f32.store 0($0), $pop3
+; NO-SIMD128-NEXT:    f32.lt $push0=, $4, $8
+; NO-SIMD128-NEXT:    f32.select $push1=, $4, $8, $pop0
+; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
+; NO-SIMD128-NEXT:    f32.lt $push2=, $3, $7
+; NO-SIMD128-NEXT:    f32.select $push3=, $3, $7, $pop2
+; NO-SIMD128-NEXT:    f32.store 8($0), $pop3
+; NO-SIMD128-NEXT:    f32.lt $push4=, $2, $6
+; NO-SIMD128-NEXT:    f32.select $push5=, $2, $6, $pop4
+; NO-SIMD128-NEXT:    f32.store 4($0), $pop5
+; NO-SIMD128-NEXT:    f32.lt $push6=, $1, $5
+; NO-SIMD128-NEXT:    f32.select $push7=, $1, $5, $pop6
+; NO-SIMD128-NEXT:    f32.store 0($0), $pop7
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: minnum_intrinsic_v4f32:
 ; NO-SIMD128-FAST:         .functype minnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    call $push0=, fminf, $1, $5
-; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fminf, $2, $6
-; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop1
-; NO-SIMD128-FAST-NEXT:    call $push2=, fminf, $3, $7
-; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
-; NO-SIMD128-FAST-NEXT:    call $push3=, fminf, $4, $8
-; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.lt $push0=, $1, $5
+; NO-SIMD128-FAST-NEXT:    f32.select $push1=, $1, $5, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
+; NO-SIMD128-FAST-NEXT:    f32.lt $push2=, $2, $6
+; NO-SIMD128-FAST-NEXT:    f32.select $push3=, $2, $6, $pop2
+; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.lt $push4=, $3, $7
+; NO-SIMD128-FAST-NEXT:    f32.select $push5=, $3, $7, $pop4
+; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop5
+; NO-SIMD128-FAST-NEXT:    f32.lt $push6=, $4, $8
+; NO-SIMD128-FAST-NEXT:    f32.select $push7=, $4, $8, $pop6
+; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop7
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float> %y)
   ret <4 x float> %a
@@ -11830,26 +11838,26 @@ define <4 x float> @minnum_nsz_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) {
 ; NO-SIMD128-LABEL: minnum_nsz_intrinsic_v4f32:
 ; NO-SIMD128:         .functype minnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    call $push0=, fminf, $4, $8
+; NO-SIMD128-NEXT:    f32.min $push0=, $4, $8
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop0
-; NO-SIMD128-NEXT:    call $push1=, fminf, $3, $7
+; NO-SIMD128-NEXT:    f32.min $push1=, $3, $7
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop1
-; NO-SIMD128-NEXT:    call $push2=, fminf, $2, $6
+; NO-SIMD128-NEXT:    f32.min $push2=, $2, $6
 ; NO-SIMD128-NEXT:    f32.store 4($0), $pop2
-; NO-SIMD128-NEXT:    call $push3=, fminf, $1, $5
+; NO-SIMD128-NEXT:    f32.min $push3=, $1, $5
 ; NO-SIMD128-NEXT:    f32.store 0($0), $pop3
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: minnum_nsz_intrinsic_v4f32:
 ; NO-SIMD128-FAST:         .functype minnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    call $push0=, fminf, $1, $5
+; NO-SIMD128-FAST-NEXT:    f32.min $push0=, $1, $5
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fminf, $2, $6
+; NO-SIMD128-FAST-NEXT:    f32.min $push1=, $2, $6
 ; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop1
-; NO-SIMD128-FAST-NEXT:    call $push2=, fminf, $3, $7
+; NO-SIMD128-FAST-NEXT:    f32.min $push2=, $3, $7
 ; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
-; NO-SIMD128-FAST-NEXT:    call $push3=, fminf, $4, $8
+; NO-SIMD128-FAST-NEXT:    f32.min $push3=, $4, $8
 ; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop3
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan nsz <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float> %y)
@@ -11875,16 +11883,16 @@ define <4 x float> @fminnumv432_non_zero_intrinsic(<4 x float> %x) {
 ; NO-SIMD128:         .functype fminnumv432_non_zero_intrinsic (i32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
 ; NO-SIMD128-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-NEXT:    call $push1=, fminf, $4, $pop0
+; NO-SIMD128-NEXT:    f32.min $push1=, $4, $pop0
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
 ; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-NEXT:    call $push2=, fminf, $3, $pop7
+; NO-SIMD128-NEXT:    f32.min $push2=, $3, $pop7
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop2
 ; NO-SIMD128-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-NEXT:    call $push3=, fminf, $2, $pop6
+; NO-SIMD128-NEXT:    f32.min $push3=, $2, $pop6
 ; NO-SIMD128-NEXT:    f32.store 4($0), $pop3
 ; NO-SIMD128-NEXT:    f32.const $push5=, -0x1p0
-; NO-SIMD128-NEXT:    call $push4=, fminf, $1, $pop5
+; NO-SIMD128-NEXT:    f32.min $push4=, $1, $pop5
 ; NO-SIMD128-NEXT:    f32.store 0($0), $pop4
 ; NO-SIMD128-NEXT:    return
 ;
@@ -11892,16 +11900,16 @@ define <4 x float> @fminnumv432_non_zero_intrinsic(<4 x float> %x) {
 ; NO-SIMD128-FAST:         .functype fminnumv432_non_zero_intrinsic (i32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
 ; NO-SIMD128-FAST-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fminf, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.min $push1=, $1, $pop0
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
 ; NO-SIMD128-FAST-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push2=, fminf, $2, $pop7
+; NO-SIMD128-FAST-NEXT:    f32.min $push2=, $2, $pop7
 ; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop2
 ; NO-SIMD128-FAST-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push3=, fminf, $3, $pop6
+; NO-SIMD128-FAST-NEXT:    f32.min $push3=, $3, $pop6
 ; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop3
 ; NO-SIMD128-FAST-NEXT:    f32.const $push5=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push4=, fminf, $4, $pop5
+; NO-SIMD128-FAST-NEXT:    f32.min $push4=, $4, $pop5
 ; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop4
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float><float -1.0, float -1.0, float -1.0, float -1.0>)
@@ -11979,34 +11987,38 @@ define <4 x float> @fminnumv432_one_zero_intrinsic(<4 x float> %x) {
 ; NO-SIMD128:         .functype fminnumv432_one_zero_intrinsic (i32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
 ; NO-SIMD128-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-NEXT:    call $push1=, fminf, $4, $pop0
+; NO-SIMD128-NEXT:    f32.min $push1=, $4, $pop0
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
-; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-NEXT:    call $push2=, fminf, $3, $pop7
+; NO-SIMD128-NEXT:    f32.const $push9=, -0x1p0
+; NO-SIMD128-NEXT:    f32.min $push2=, $3, $pop9
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop2
 ; NO-SIMD128-NEXT:    f32.const $push3=, 0x0p0
-; NO-SIMD128-NEXT:    call $push4=, fminf, $2, $pop3
-; NO-SIMD128-NEXT:    f32.store 4($0), $pop4
-; NO-SIMD128-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-NEXT:    call $push5=, fminf, $1, $pop6
-; NO-SIMD128-NEXT:    f32.store 0($0), $pop5
+; NO-SIMD128-NEXT:    f32.const $push8=, 0x0p0
+; NO-SIMD128-NEXT:    f32.lt $push4=, $2, $pop8
+; NO-SIMD128-NEXT:    f32.select $push5=, $2, $pop3, $pop4
+; NO-SIMD128-NEXT:    f32.store 4($0), $pop5
+; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
+; NO-SIMD128-NEXT:    f32.min $push6=, $1, $pop7
+; NO-SIMD128-NEXT:    f32.store 0($0), $pop6
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: fminnumv432_one_zero_intrinsic:
 ; NO-SIMD128-FAST:         .functype fminnumv432_one_zero_intrinsic (i32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
 ; NO-SIMD128-FAST-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fminf, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.min $push1=, $1, $pop0
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
-; NO-SIMD128-FAST-NEXT:    f32.const $push2=, 0x0p0
-; NO-SIMD128-FAST-NEXT:    call $push3=, fminf, $2, $pop2
-; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.const $push9=, -0x1p0
+; NO-SIMD128-FAST-NEXT:    f32.min $push2=, $3, $pop9
+; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
+; NO-SIMD128-FAST-NEXT:    f32.const $push3=, 0x0p0
+; NO-SIMD128-FAST-NEXT:    f32.const $push8=, 0x0p0
+; NO-SIMD128-FAST-NEXT:    f32.lt $push4=, $2, $pop8
+; NO-SIMD128-FAST-NEXT:    f32.select $push5=, $2, $pop3, $pop4
+; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop5
 ; NO-SIMD128-FAST-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push4=, fminf, $3, $pop7
-; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop4
-; NO-SIMD128-FAST-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push5=, fminf, $4, $pop6
-; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop5
+; NO-SIMD128-FAST-NEXT:    f32.min $push6=, $4, $pop7
+; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop6
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float><float -1.0, float 0.0, float -1.0, float -1.0>)
   ret <4 x float> %a
@@ -12126,27 +12138,35 @@ define <4 x float> @maxnum_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) {
 ; NO-SIMD128-LABEL: maxnum_intrinsic_v4f32:
 ; NO-SIMD128:         .functype maxnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    call $push0=, fmaxf, $4, $8
-; NO-SIMD128-NEXT:    f32.store 12($0), $pop0
-; NO-SIMD128-NEXT:    call $push1=, fmaxf, $3, $7
-; NO-SIMD128-NEXT:    f32.store 8($0), $pop1
-; NO-SIMD128-NEXT:    call $push2=, fmaxf, $2, $6
-; NO-SIMD128-NEXT:    f32.store 4($0), $pop2
-; NO-SIMD128-NEXT:    call $push3=, fmaxf, $1, $5
-; NO-SIMD128-NEXT:    f32.store 0($0), $pop3
+; NO-SIMD128-NEXT:    f32.gt $push0=, $4, $8
+; NO-SIMD128-NEXT:    f32.select $push1=, $4, $8, $pop0
+; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
+; NO-SIMD128-NEXT:    f32.gt $push2=, $3, $7
+; NO-SIMD128-NEXT:    f32.select $push3=, $3, $7, $pop2
+; NO-SIMD128-NEXT:    f32.store 8($0), $pop3
+; NO-SIMD128-NEXT:    f32.gt $push4=, $2, $6
+; NO-SIMD128-NEXT:    f32.select $push5=, $2, $6, $pop4
+; NO-SIMD128-NEXT:    f32.store 4($0), $pop5
+; NO-SIMD128-NEXT:    f32.gt $push6=, $1, $5
+; NO-SIMD128-NEXT:    f32.select $push7=, $1, $5, $pop6
+; NO-SIMD128-NEXT:    f32.store 0($0), $pop7
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: maxnum_intrinsic_v4f32:
 ; NO-SIMD128-FAST:         .functype maxnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    call $push0=, fmaxf, $1, $5
-; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fmaxf, $2, $6
-; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop1
-; NO-SIMD128-FAST-NEXT:    call $push2=, fmaxf, $3, $7
-; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
-; NO-SIMD128-FAST-NEXT:    call $push3=, fmaxf, $4, $8
-; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.gt $push0=, $1, $5
+; NO-SIMD128-FAST-NEXT:    f32.select $push1=, $1, $5, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
+; NO-SIMD128-FAST-NEXT:    f32.gt $push2=, $2, $6
+; NO-SIMD128-FAST-NEXT:    f32.select $push3=, $2, $6, $pop2
+; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.gt $push4=, $3, $7
+; NO-SIMD128-FAST-NEXT:    f32.select $push5=, $3, $7, $pop4
+; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop5
+; NO-SIMD128-FAST-NEXT:    f32.gt $push6=, $4, $8
+; NO-SIMD128-FAST-NEXT:    f32.select $push7=, $4, $8, $pop6
+; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop7
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float> %y)
   ret <4 x float> %a
@@ -12168,26 +12188,26 @@ define <4 x float> @maxnum_nsz_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) {
 ; NO-SIMD128-LABEL: maxnum_nsz_intrinsic_v4f32:
 ; NO-SIMD128:         .functype maxnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    call $push0=, fmaxf, $4, $8
+; NO-SIMD128-NEXT:    f32.max $push0=, $4, $8
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop0
-; NO-SIMD128-NEXT:    call $push1=, fmaxf, $3, $7
+; NO-SIMD128-NEXT:    f32.max $push1=, $3, $7
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop1
-; NO-SIMD128-NEXT:    call $push2=, fmaxf, $2, $6
+; NO-SIMD128-NEXT:    f32.max $push2=, $2, $6
 ; NO-SIMD128-NEXT:    f32.store 4($0), $pop2
-; NO-SIMD128-NEXT:    call $push3=, fmaxf, $1, $5
+; NO-SIMD128-NEXT:    f32.max $push3=, $1, $5
 ; NO-SIMD128-NEXT:    f32.store 0($0), $pop3
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: maxnum_nsz_intrinsic_v4f32:
 ; NO-SIMD128-FAST:         .functype maxnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    call $push0=, fmaxf, $1, $5
+; NO-SIMD128-FAST-NEXT:    f32.max $push0=, $1, $5
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fmaxf, $2, $6
+; NO-SIMD128-FAST-NEXT:    f32.max $push1=, $2, $6
 ; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop1
-; NO-SIMD128-FAST-NEXT:    call $push2=, fmaxf, $3, $7
+; NO-SIMD128-FAST-NEXT:    f32.max $push2=, $3, $7
 ; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
-; NO-SIMD128-FAST-NEXT:    call $push3=, fmaxf, $4, $8
+; NO-SIMD128-FAST-NEXT:    f32.max $push3=, $4, $8
 ; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop3
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float> %y)
@@ -12265,34 +12285,38 @@ define <4 x float> @maxnum_one_zero_intrinsic_v4f32(<4 x float> %x, <4 x float>
 ; NO-SIMD128:         .functype maxnum_one_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
 ; NO-SIMD128-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-NEXT:    call $push1=, fmaxf, $4, $pop0
+; NO-SIMD128-NEXT:    f32.max $push1=, $4, $pop0
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
-; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-NEXT:    call $push2=, fmaxf, $3, $pop7
+; NO-SIMD128-NEXT:    f32.const $push9=, -0x1p0
+; NO-SIMD128-NEXT:    f32.max $push2=, $3, $pop9
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop2
 ; NO-SIMD128-NEXT:    f32.const $push3=, 0x0p0
-; NO-SIMD128-NEXT:    call $push4=, fmaxf, $2, $pop3
-; NO-SIMD128-NEXT:    f32.store 4($0), $pop4
-; NO-SIMD128-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-NEXT:    call $push5=, fmaxf, $1, $pop6
-; NO-SIMD128-NEXT:    f32.store 0($0), $pop5
+; NO-SIMD128-NEXT:    f32.const $push8=, 0x0p0
+; NO-SIMD128-NEXT:    f32.gt $push4=, $2, $pop8
+; NO-SIMD128-NEXT:    f32.select $push5=, $2, $pop3, $pop4
+; NO-SIMD128-NEXT:    f32.store 4($0), $pop5
+; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
+; NO-SIMD128-NEXT:    f32.max $push6=, $1, $pop7
+; NO-SIMD128-NEXT:    f32.store 0($0), $pop6
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: maxnum_one_zero_intrinsic_v4f32:
 ; NO-SIMD128-FAST:         .functype maxnum_one_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
 ; NO-SIMD128-FAST-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fmaxf, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.max $push1=, $1, $pop0
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
-; NO-SIMD128-FAST-NEXT:    f32.const $push2=, 0x0p0
-; NO-SIMD128-FAST-NEXT:    call $push3=, fmaxf, $2, $pop2
-; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop3
+; NO-SIMD128-FAST-NEXT:    f32.const $push9=, -0x1p0
+; NO-SIMD128-FAST-NEXT:    f32.max $push2=, $3, $pop9
+; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop2
+; NO-SIMD128-FAST-NEXT:    f32.const $push3=, 0x0p0
+; NO-SIMD128-FAST-NEXT:    f32.const $push8=, 0x0p0
+; NO-SIMD128-FAST-NEXT:    f32.gt $push4=, $2, $pop8
+; NO-SIMD128-FAST-NEXT:    f32.select $push5=, $2, $pop3, $pop4
+; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop5
 ; NO-SIMD128-FAST-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push4=, fmaxf, $3, $pop7
-; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop4
-; NO-SIMD128-FAST-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push5=, fmaxf, $4, $pop6
-; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop5
+; NO-SIMD128-FAST-NEXT:    f32.max $push6=, $4, $pop7
+; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop6
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float><float -1.0, float 0.0, float -1.0, float -1.0>)
   ret <4 x float> %a
@@ -12317,16 +12341,16 @@ define <4 x float> @maxnum_non_zero_intrinsic_v4f32(<4 x float> %x, <4 x float>
 ; NO-SIMD128:         .functype maxnum_non_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
 ; NO-SIMD128-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-NEXT:    call $push1=, fmaxf, $4, $pop0
+; NO-SIMD128-NEXT:    f32.max $push1=, $4, $pop0
 ; NO-SIMD128-NEXT:    f32.store 12($0), $pop1
 ; NO-SIMD128-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-NEXT:    call $push2=, fmaxf, $3, $pop7
+; NO-SIMD128-NEXT:    f32.max $push2=, $3, $pop7
 ; NO-SIMD128-NEXT:    f32.store 8($0), $pop2
 ; NO-SIMD128-NEXT:    f32.const $push3=, 0x1p0
-; NO-SIMD128-NEXT:    call $push4=, fmaxf, $2, $pop3
+; NO-SIMD128-NEXT:    f32.max $push4=, $2, $pop3
 ; NO-SIMD128-NEXT:    f32.store 4($0), $pop4
 ; NO-SIMD128-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-NEXT:    call $push5=, fmaxf, $1, $pop6
+; NO-SIMD128-NEXT:    f32.max $push5=, $1, $pop6
 ; NO-SIMD128-NEXT:    f32.store 0($0), $pop5
 ; NO-SIMD128-NEXT:    return
 ;
@@ -12334,16 +12358,16 @@ define <4 x float> @maxnum_non_zero_intrinsic_v4f32(<4 x float> %x, <4 x float>
 ; NO-SIMD128-FAST:         .functype maxnum_non_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
 ; NO-SIMD128-FAST-NEXT:    f32.const $push0=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push1=, fmaxf, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    f32.max $push1=, $1, $pop0
 ; NO-SIMD128-FAST-NEXT:    f32.store 0($0), $pop1
 ; NO-SIMD128-FAST-NEXT:    f32.const $push2=, 0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push3=, fmaxf, $2, $pop2
+; NO-SIMD128-FAST-NEXT:    f32.max $push3=, $2, $pop2
 ; NO-SIMD128-FAST-NEXT:    f32.store 4($0), $pop3
 ; NO-SIMD128-FAST-NEXT:    f32.const $push7=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push4=, fmaxf, $3, $pop7
+; NO-SIMD128-FAST-NEXT:    f32.max $push4=, $3, $pop7
 ; NO-SIMD128-FAST-NEXT:    f32.store 8($0), $pop4
 ; NO-SIMD128-FAST-NEXT:    f32.const $push6=, -0x1p0
-; NO-SIMD128-FAST-NEXT:    call $push5=, fmaxf, $4, $pop6
+; NO-SIMD128-FAST-NEXT:    f32.max $push5=, $4, $pop6
 ; NO-SIMD128-FAST-NEXT:    f32.store 12($0), $pop5
 ; NO-SIMD128-FAST-NEXT:    return
   %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float><float -1.0, float 1.0, float -1.0, float -1.0>)
diff --git a/llvm/test/CodeGen/WebAssembly/vararg-frame.ll b/llvm/test/CodeGen/WebAssembly/vararg-frame.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5c76040325cc94da97e99ab4ea796f5ba3271c64
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/vararg-frame.ll
@@ -0,0 +1,526 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs  | FileCheck %s
+; REQUIRES: webassembly-registered-target
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+; Function Attrs: nounwind
+define void @pass_s0() {
+; CHECK-LABEL: pass_s0:
+; CHECK:         .functype pass_s0 () -> ()
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink()
+  ret void
+}
+
+declare void @sink(...)
+
+; Function Attrs: nounwind
+define void @pass_s1(i8 %x.coerce) {
+; CHECK-LABEL: pass_s1:
+; CHECK:         .functype pass_s1 (i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 1
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i8 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_s2(i16 %x.coerce) {
+; CHECK-LABEL: pass_s2:
+; CHECK:         .functype pass_s2 (i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 1
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i16 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_s3(i32 %x.coerce) {
+; CHECK-LABEL: pass_s3:
+; CHECK:         .functype pass_s3 (i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 1
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_s4(i64 %x.coerce) {
+; CHECK-LABEL: pass_s4:
+; CHECK:         .functype pass_s4 (i64) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 1
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i64 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_s5(<4 x i32> noundef %x) {
+; CHECK-LABEL: pass_s5:
+; CHECK:         .functype pass_s5 (i32, i32, i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 4
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i32.store 12
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.store 8
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(<4 x i32> noundef %x)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s0(i32 noundef %i) {
+; CHECK-LABEL: pass_int_s0:
+; CHECK:         .functype pass_int_s0 (i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 1
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s1(i32 noundef %i, i8 %x.coerce) {
+; CHECK-LABEL: pass_int_s1:
+; CHECK:         .functype pass_int_s1 (i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 2
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, i8 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s2(i32 noundef %i, i16 %x.coerce) {
+; CHECK-LABEL: pass_int_s2:
+; CHECK:         .functype pass_int_s2 (i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 2
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, i16 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s3(i32 noundef %i, i32 %x.coerce) {
+; CHECK-LABEL: pass_int_s3:
+; CHECK:         .functype pass_int_s3 (i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 2
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, i32 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s4(i32 noundef %i, i64 %x.coerce) {
+; CHECK-LABEL: pass_int_s4:
+; CHECK:         .functype pass_int_s4 (i32, i64) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 2
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, i64 %x.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_int_s5(i32 noundef %i, <4 x i32> noundef %x) {
+; CHECK-LABEL: pass_int_s5:
+; CHECK:         .functype pass_int_s5 (i32, i32, i32, i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 5
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.const 28
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.const 20
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, <4 x i32> noundef %x)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_asc(i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5) {
+; CHECK-LABEL: pass_asc:
+; CHECK:         .functype pass_asc (i32, i32, i32, i64, i32, i32, i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 8
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 44
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 7
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 40
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 6
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 36
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.store 8
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_dsc(<4 x i32> noundef %x0, i64 %x1.coerce, i32 %x2.coerce, i16 %x3.coerce, i8 %x4.coerce) {
+; CHECK-LABEL: pass_dsc:
+; CHECK:         .functype pass_dsc (i32, i32, i32, i32, i64, i32, i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 8
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 7
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 28
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 6
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i32.store 12
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.store 8
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(<4 x i32> noundef %x0, i64 %x1.coerce, i32 %x2.coerce, i16 %x3.coerce, i8 %x4.coerce)
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @pass_multiple(i32 noundef %i, i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5) {
+; CHECK-LABEL: pass_multiple:
+; CHECK:         .functype pass_multiple (i32, i32, i32, i32, i64, i32, i32, i32, i32) -> ()
+; CHECK-NEXT:    .local i32
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    global.get __stack_pointer
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.sub
+; CHECK-NEXT:    local.tee 9
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 4
+; CHECK-NEXT:    i64.store 40
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.store 36
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 32
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 28
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 8
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 7
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 20
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 6
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 5
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 3
+; CHECK-NEXT:    i32.store 8
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.store 4
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.store 0
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    call sink
+; CHECK-NEXT:    local.get 9
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    global.set __stack_pointer
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  tail call void (...) @sink(i32 noundef %i, i16 %x2.coerce, i64 %x4.coerce)
+  tail call void (...) @sink(i32 noundef %i, i8 %x1.coerce, i32 %x3.coerce, <4 x i32> noundef %x5)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/X86/atomic-fp.ll b/llvm/test/CodeGen/X86/atomic-fp.ll
index 1094edd19af4380465a94095ae7817cf1d8aeb69..fe79dfe39f6452d07b060ab2223a6b45cca08b05 100644
--- a/llvm/test/CodeGen/X86/atomic-fp.ll
+++ b/llvm/test/CodeGen/X86/atomic-fp.ll
@@ -777,3 +777,2326 @@ bb:
   store atomic i64 %tmp9, ptr %tmp4 monotonic, align 8
   ret void
 }
+
+; ----- FSUB -----
+
+define dso_local void @fsub_32r(ptr %loc, float %val) nounwind {
+; X86-NOSSE-LABEL: fsub_32r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl (%eax), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fsubs {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%eax)
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_32r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl (%eax), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    subss {{[0-9]+}}(%esp), %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_32r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    subss {{[0-9]+}}(%esp), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_32r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vsubss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_32r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    subss %xmm0, %xmm1
+; X64-SSE-NEXT:    movss %xmm1, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_32r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i32, ptr %loc seq_cst, align 4
+  %2 = bitcast i32 %1 to float
+  %sub = fsub float %2, %val
+  %3 = bitcast float %sub to i32
+  store atomic i32 %3, ptr %loc release, align 4
+  ret void
+}
+
+define dso_local void @fsub_64r(ptr %loc, double %val) nounwind {
+; X86-NOSSE-LABEL: fsub_64r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    movl 8(%ebp), %eax
+; X86-NOSSE-NEXT:    fildll (%eax)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fsubl 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%eax)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_64r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 8(%ebp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fsubl 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_64r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 8(%ebp), %eax
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    subsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_64r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 8(%ebp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vsubsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%eax)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_64r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE-NEXT:    subsd %xmm0, %xmm1
+; X64-SSE-NEXT:    movsd %xmm1, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_64r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i64, ptr %loc seq_cst, align 8
+  %2 = bitcast i64 %1 to double
+  %sub = fsub double %2, %val
+  %3 = bitcast double %sub to i64
+  store atomic i64 %3, ptr %loc release, align 8
+  ret void
+}
+
+; Floating-point sub to a global using an immediate.
+define dso_local void @fsub_32g() nounwind {
+; X86-NOSSE-LABEL: fsub_32g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl glob32, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fchs
+; X86-NOSSE-NEXT:    fadds (%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, glob32
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_32g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl glob32, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    addss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, glob32
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_32g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    addss glob32, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, glob32
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_32g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vaddss glob32, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, glob32
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_32g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    addss glob32(%rip), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, glob32(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_32g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vaddss glob32(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, glob32(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr @glob32 monotonic, align 4
+  %f = bitcast i32 %i to float
+  %sub = fsub float %f, 1.000000e+00
+  %s = bitcast float %sub to i32
+  store atomic i32 %s, ptr @glob32 monotonic, align 4
+  ret void
+}
+
+define dso_local void @fsub_64g() nounwind {
+; X86-NOSSE-LABEL: fsub_64g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll glob64
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fchs
+; X86-NOSSE-NEXT:    faddl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll glob64
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_64g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fld1
+; X86-SSE1-NEXT:    fchs
+; X86-SSE1-NEXT:    faddl (%esp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, glob64
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_64g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    addsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, glob64
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_64g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vaddsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, glob64
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_64g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0]
+; X64-SSE-NEXT:    addsd glob64(%rip), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, glob64(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_64g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vaddsd glob64(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, glob64(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr @glob64 monotonic, align 8
+  %f = bitcast i64 %i to double
+  %sub = fsub double %f, 1.000000e+00
+  %s = bitcast double %sub to i64
+  store atomic i64 %s, ptr @glob64 monotonic, align 8
+  ret void
+}
+
+; Floating-point sub to a hard-coded immediate location using an immediate.
+define dso_local void @fsub_32imm() nounwind {
+; X86-NOSSE-LABEL: fsub_32imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl -559038737, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fchs
+; X86-NOSSE-NEXT:    fadds (%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, -559038737
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_32imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl -559038737, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    addss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, -559038737
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_32imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    addss -559038737, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, -559038737
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_32imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vaddss -559038737, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, -559038737
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_32imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    addss (%rax), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_32imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vaddss (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  %f = bitcast i32 %i to float
+  %sub = fsub float %f, 1.000000e+00
+  %s = bitcast float %sub to i32
+  store atomic i32 %s, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  ret void
+}
+
+define dso_local void @fsub_64imm() nounwind {
+; X86-NOSSE-LABEL: fsub_64imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll -559038737
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fchs
+; X86-NOSSE-NEXT:    faddl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll -559038737
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_64imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fld1
+; X86-SSE1-NEXT:    fchs
+; X86-SSE1-NEXT:    faddl (%esp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, -559038737
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_64imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    addsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, -559038737
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_64imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vaddsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, -559038737
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_64imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0]
+; X64-SSE-NEXT:    addsd (%rax), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_64imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vaddsd (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  %f = bitcast i64 %i to double
+  %sub = fsub double %f, 1.000000e+00
+  %s = bitcast double %sub to i64
+  store atomic i64 %s, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  ret void
+}
+
+; Floating-point sub to a stack location.
+define dso_local void @fsub_32stack() nounwind {
+; X86-NOSSE-LABEL: fsub_32stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $12, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fsubs (%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    addl $12, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_32stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $12, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE1-NEXT:    subss (%esp), %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    addl $12, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_32stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    subss (%esp), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%esp)
+; X86-SSE2-NEXT:    popl %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_32stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vsubss (%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
+; X86-AVX-NEXT:    popl %eax
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_32stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    subss -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_32stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vsubss -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i32, align 4
+  %load = load atomic i32, ptr %ptr acquire, align 4
+  %bc0 = bitcast i32 %load to float
+  %fsub = fsub float 1.000000e+00, %bc0
+  %bc1 = bitcast float %fsub to i32
+  store atomic i32 %bc1, ptr %ptr release, align 4
+  ret void
+}
+
+define dso_local void @fsub_64stack() nounwind {
+; X86-NOSSE-LABEL: fsub_64stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fsubl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_64stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $24, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fld1
+; X86-SSE1-NEXT:    fsubl (%esp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_64stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    subsd %xmm0, %xmm1
+; X86-SSE2-NEXT:    movsd %xmm1, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_64stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_64stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; X64-SSE-NEXT:    subsd -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_64stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vsubsd -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i64, align 8
+  %load = load atomic i64, ptr %ptr acquire, align 8
+  %bc0 = bitcast i64 %load to double
+  %fsub = fsub double 1.000000e+00, %bc0
+  %bc1 = bitcast double %fsub to i64
+  store atomic i64 %bc1, ptr %ptr release, align 8
+  ret void
+}
+
+define dso_local void @fsub_array(ptr %arg, double %arg1, i64 %arg2) nounwind {
+; X86-NOSSE-LABEL: fsub_array:
+; X86-NOSSE:       # %bb.0: # %bb
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    movl 20(%ebp), %eax
+; X86-NOSSE-NEXT:    movl 8(%ebp), %ecx
+; X86-NOSSE-NEXT:    fildll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fsubl 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %edx, (%esp)
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    leal -4(%ebp), %esp
+; X86-NOSSE-NEXT:    popl %esi
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fsub_array:
+; X86-SSE1:       # %bb.0: # %bb
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 20(%ebp), %eax
+; X86-SSE1-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fsubl 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fsub_array:
+; X86-SSE2:       # %bb.0: # %bb
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 20(%ebp), %eax
+; X86-SSE2-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    subsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fsub_array:
+; X86-AVX:       # %bb.0: # %bb
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 20(%ebp), %eax
+; X86-AVX-NEXT:    movl 8(%ebp), %ecx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vsubsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%ecx,%eax,8)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fsub_array:
+; X64-SSE:       # %bb.0: # %bb
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE-NEXT:    subsd %xmm0, %xmm1
+; X64-SSE-NEXT:    movsd %xmm1, (%rdi,%rsi,8)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fsub_array:
+; X64-AVX:       # %bb.0: # %bb
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi,%rsi,8)
+; X64-AVX-NEXT:    retq
+bb:
+  %tmp4 = getelementptr inbounds i64, ptr %arg, i64 %arg2
+  %tmp6 = load atomic i64, ptr %tmp4 monotonic, align 8
+  %tmp7 = bitcast i64 %tmp6 to double
+  %tmp8 = fsub double %tmp7, %arg1
+  %tmp9 = bitcast double %tmp8 to i64
+  store atomic i64 %tmp9, ptr %tmp4 monotonic, align 8
+  ret void
+}
+
+; ----- FMUL -----
+
+define dso_local void @fmul_32r(ptr %loc, float %val) nounwind {
+; X86-NOSSE-LABEL: fmul_32r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl (%eax), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fmuls {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%eax)
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_32r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl (%eax), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    mulss {{[0-9]+}}(%esp), %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_32r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    mulss (%eax), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_32r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmulss (%eax), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_32r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    mulss (%rdi), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_32r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmulss (%rdi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i32, ptr %loc seq_cst, align 4
+  %2 = bitcast i32 %1 to float
+  %mul = fmul float %2, %val
+  %3 = bitcast float %mul to i32
+  store atomic i32 %3, ptr %loc release, align 4
+  ret void
+}
+
+define dso_local void @fmul_64r(ptr %loc, double %val) nounwind {
+; X86-NOSSE-LABEL: fmul_64r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    movl 8(%ebp), %eax
+; X86-NOSSE-NEXT:    fildll (%eax)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fmull 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%eax)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_64r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 8(%ebp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fmull 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_64r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 8(%ebp), %eax
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    mulsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_64r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 8(%ebp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%eax)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_64r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    mulsd (%rdi), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_64r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmulsd (%rdi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i64, ptr %loc seq_cst, align 8
+  %2 = bitcast i64 %1 to double
+  %mul = fmul double %2, %val
+  %3 = bitcast double %mul to i64
+  store atomic i64 %3, ptr %loc release, align 8
+  ret void
+}
+
+; Floating-point mul to a global using an immediate.
+define dso_local void @fmul_32g() nounwind {
+; X86-NOSSE-LABEL: fmul_32g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl glob32, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, glob32
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_32g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl glob32, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, glob32
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_32g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    mulss glob32, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, glob32
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_32g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vmulss glob32, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, glob32
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_32g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    mulss glob32(%rip), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, glob32(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_32g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulss glob32(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, glob32(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr @glob32 monotonic, align 4
+  %f = bitcast i32 %i to float
+  %mul = fmul float %f, 0x400921FA00000000
+  %s = bitcast float %mul to i32
+  store atomic i32 %s, ptr @glob32 monotonic, align 4
+  ret void
+}
+
+define dso_local void @fmul_64g() nounwind {
+; X86-NOSSE-LABEL: fmul_64g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll glob64
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll glob64
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_64g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, glob64
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_64g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, glob64
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_64g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, glob64
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_64g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-SSE-NEXT:    mulsd glob64(%rip), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, glob64(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_64g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulsd glob64(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, glob64(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr @glob64 monotonic, align 8
+  %f = bitcast i64 %i to double
+  %mul = fmul double %f, 0x400921FA00000000
+  %s = bitcast double %mul to i64
+  store atomic i64 %s, ptr @glob64 monotonic, align 8
+  ret void
+}
+
+; Floating-point mul to a hard-coded immediate location using an immediate.
+define dso_local void @fmul_32imm() nounwind {
+; X86-NOSSE-LABEL: fmul_32imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl -559038737, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, -559038737
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_32imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl -559038737, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, -559038737
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_32imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    mulss -559038737, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, -559038737
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_32imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vmulss -559038737, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, -559038737
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_32imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    mulss (%rax), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_32imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulss (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  %f = bitcast i32 %i to float
+  %mul = fmul float %f, 0x400921FA00000000
+  %s = bitcast float %mul to i32
+  store atomic i32 %s, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  ret void
+}
+
+define dso_local void @fmul_64imm() nounwind {
+; X86-NOSSE-LABEL: fmul_64imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll -559038737
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll -559038737
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_64imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, -559038737
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_64imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, -559038737
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_64imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, -559038737
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_64imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-SSE-NEXT:    mulsd (%rax), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_64imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulsd (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  %f = bitcast i64 %i to double
+  %mul = fmul double %f, 0x400921FA00000000
+  %s = bitcast double %mul to i64
+  store atomic i64 %s, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  ret void
+}
+
+; Floating-point mul to a stack location.
+define dso_local void @fmul_32stack() nounwind {
+; X86-NOSSE-LABEL: fmul_32stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $12, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    addl $12, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_32stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $12, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    addl $12, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_32stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    mulss (%esp), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%esp)
+; X86-SSE2-NEXT:    popl %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_32stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vmulss (%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
+; X86-AVX-NEXT:    popl %eax
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_32stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    mulss -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_32stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [3.14159012E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulss -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i32, align 4
+  %load = load atomic i32, ptr %ptr acquire, align 4
+  %bc0 = bitcast i32 %load to float
+  %fmul = fmul float 0x400921FA00000000, %bc0
+  %bc1 = bitcast float %fmul to i32
+  store atomic i32 %bc1, ptr %ptr release, align 4
+  ret void
+}
+
+define dso_local void @fmul_64stack() nounwind {
+; X86-NOSSE-LABEL: fmul_64stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_64stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $24, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fmuls {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_64stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_64stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_64stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-SSE-NEXT:    mulsd -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_64stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [3.1415901184082031E+0,0.0E+0]
+; X64-AVX-NEXT:    vmulsd -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i64, align 8
+  %load = load atomic i64, ptr %ptr acquire, align 8
+  %bc0 = bitcast i64 %load to double
+  %fmul = fmul double 0x400921FA00000000, %bc0
+  %bc1 = bitcast double %fmul to i64
+  store atomic i64 %bc1, ptr %ptr release, align 8
+  ret void
+}
+
+define dso_local void @fmul_array(ptr %arg, double %arg1, i64 %arg2) nounwind {
+; X86-NOSSE-LABEL: fmul_array:
+; X86-NOSSE:       # %bb.0: # %bb
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    movl 20(%ebp), %eax
+; X86-NOSSE-NEXT:    movl 8(%ebp), %ecx
+; X86-NOSSE-NEXT:    fildll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fmull 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %edx, (%esp)
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    leal -4(%ebp), %esp
+; X86-NOSSE-NEXT:    popl %esi
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fmul_array:
+; X86-SSE1:       # %bb.0: # %bb
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 20(%ebp), %eax
+; X86-SSE1-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fmull 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fmul_array:
+; X86-SSE2:       # %bb.0: # %bb
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 20(%ebp), %eax
+; X86-SSE2-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    mulsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fmul_array:
+; X86-AVX:       # %bb.0: # %bb
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 20(%ebp), %eax
+; X86-AVX-NEXT:    movl 8(%ebp), %ecx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmulsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%ecx,%eax,8)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fmul_array:
+; X64-SSE:       # %bb.0: # %bb
+; X64-SSE-NEXT:    mulsd (%rdi,%rsi,8), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rdi,%rsi,8)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fmul_array:
+; X64-AVX:       # %bb.0: # %bb
+; X64-AVX-NEXT:    vmulsd (%rdi,%rsi,8), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi,%rsi,8)
+; X64-AVX-NEXT:    retq
+bb:
+  %tmp4 = getelementptr inbounds i64, ptr %arg, i64 %arg2
+  %tmp6 = load atomic i64, ptr %tmp4 monotonic, align 8
+  %tmp7 = bitcast i64 %tmp6 to double
+  %tmp8 = fmul double %tmp7, %arg1
+  %tmp9 = bitcast double %tmp8 to i64
+  store atomic i64 %tmp9, ptr %tmp4 monotonic, align 8
+  ret void
+}
+
+; ----- FDIV -----
+
+define dso_local void @fdiv_32r(ptr %loc, float %val) nounwind {
+; X86-NOSSE-LABEL: fdiv_32r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl (%eax), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fdivs {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, (%eax)
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_32r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl (%eax), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    divss {{[0-9]+}}(%esp), %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_32r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    divss {{[0-9]+}}(%esp), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_32r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vdivss {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_32r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    divss %xmm0, %xmm1
+; X64-SSE-NEXT:    movss %xmm1, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_32r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vdivss %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i32, ptr %loc seq_cst, align 4
+  %2 = bitcast i32 %1 to float
+  %div = fdiv float %2, %val
+  %3 = bitcast float %div to i32
+  store atomic i32 %3, ptr %loc release, align 4
+  ret void
+}
+
+define dso_local void @fdiv_64r(ptr %loc, double %val) nounwind {
+; X86-NOSSE-LABEL: fdiv_64r:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    movl 8(%ebp), %eax
+; X86-NOSSE-NEXT:    fildll (%eax)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fdivl 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl %ecx, (%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%eax)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_64r:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 8(%ebp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fdivl 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_64r:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 8(%ebp), %eax
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    divsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_64r:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 8(%ebp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vdivsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%eax)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_64r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE-NEXT:    divsd %xmm0, %xmm1
+; X64-SSE-NEXT:    movsd %xmm1, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_64r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
+  %1 = load atomic i64, ptr %loc seq_cst, align 8
+  %2 = bitcast i64 %1 to double
+  %div = fdiv double %2, %val
+  %3 = bitcast double %div to i64
+  store atomic i64 %3, ptr %loc release, align 8
+  ret void
+}
+
+; Floating-point div to a global using an immediate.
+define dso_local void @fdiv_32g() nounwind {
+; X86-NOSSE-LABEL: fdiv_32g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl glob32, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, glob32
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_32g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl glob32, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, glob32
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_32g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, glob32
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_32g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vdivss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, glob32
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_32g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, glob32(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_32g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vdivss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, glob32(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr @glob32 monotonic, align 4
+  %f = bitcast i32 %i to float
+  %div = fdiv float %f, 0x400921FA00000000
+  %s = bitcast float %div to i32
+  store atomic i32 %s, ptr @glob32 monotonic, align 4
+  ret void
+}
+
+define dso_local void @fdiv_64g() nounwind {
+; X86-NOSSE-LABEL: fdiv_64g:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll glob64
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll glob64
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_64g:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, glob64
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_64g:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    divsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, glob64
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_64g:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vdivsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, glob64
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_64g:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE-NEXT:    divsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, glob64(%rip)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_64g:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-AVX-NEXT:    vdivsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, glob64(%rip)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr @glob64 monotonic, align 8
+  %f = bitcast i64 %i to double
+  %div = fdiv double %f, 0x400921FA00000000
+  %s = bitcast double %div to i64
+  store atomic i64 %s, ptr @glob64 monotonic, align 8
+  ret void
+}
+
+; Floating-point div to a hard-coded immediate location using an immediate.
+define dso_local void @fdiv_32imm() nounwind {
+; X86-NOSSE-LABEL: fdiv_32imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $8, %esp
+; X86-NOSSE-NEXT:    movl -559038737, %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    flds (%esp)
+; X86-NOSSE-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, -559038737
+; X86-NOSSE-NEXT:    addl $8, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_32imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    movl -559038737, %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, -559038737
+; X86-SSE1-NEXT:    addl $8, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_32imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, -559038737
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_32imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vdivss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, -559038737
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_32imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-SSE-NEXT:    divss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_32imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vdivss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i32, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  %f = bitcast i32 %i to float
+  %div = fdiv float %f, 0x400921FA00000000
+  %s = bitcast float %div to i32
+  store atomic i32 %s, ptr inttoptr (i32 3735928559 to ptr) monotonic, align 4
+  ret void
+}
+
+define dso_local void @fdiv_64imm() nounwind {
+; X86-NOSSE-LABEL: fdiv_64imm:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $32, %esp
+; X86-NOSSE-NEXT:    fildll -559038737
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll -559038737
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_64imm:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fdivs {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, -559038737
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_64imm:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    divsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, -559038737
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_64imm:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vdivsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, -559038737
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_64imm:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-SSE-NEXT:    divsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rax)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_64imm:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-AVX-NEXT:    vdivsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rax)
+; X64-AVX-NEXT:    retq
+  %i = load atomic i64, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  %f = bitcast i64 %i to double
+  %div = fdiv double %f, 0x400921FA00000000
+  %s = bitcast double %div to i64
+  store atomic i64 %s, ptr inttoptr (i64 3735928559 to ptr) monotonic, align 8
+  ret void
+}
+
+; Floating-point div to a stack location.
+define dso_local void @fdiv_32stack() nounwind {
+; X86-NOSSE-LABEL: fdiv_32stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    subl $12, %esp
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fdivs (%esp)
+; X86-NOSSE-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    addl $12, %esp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_32stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    subl $12, %esp
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, (%esp)
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE1-NEXT:    divss (%esp), %xmm0
+; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    addl $12, %esp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_32stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    divss (%esp), %xmm0
+; X86-SSE2-NEXT:    movss %xmm0, (%esp)
+; X86-SSE2-NEXT:    popl %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_32stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vdivss (%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
+; X86-AVX-NEXT:    popl %eax
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_32stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-SSE-NEXT:    divss -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_32stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vdivss -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i32, align 4
+  %load = load atomic i32, ptr %ptr acquire, align 4
+  %bc0 = bitcast i32 %load to float
+  %fdiv = fdiv float 1.000000e+00, %bc0
+  %bc1 = bitcast float %fdiv to i32
+  store atomic i32 %bc1, ptr %ptr release, align 4
+  ret void
+}
+
+define dso_local void @fdiv_64stack() nounwind {
+; X86-NOSSE-LABEL: fdiv_64stack:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    fildll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fld1
+; X86-NOSSE-NEXT:    fdivl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_64stack:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $24, %esp
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fld1
+; X86-SSE1-NEXT:    fdivl (%esp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_64stack:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    divsd %xmm0, %xmm1
+; X86-SSE2-NEXT:    movsd %xmm1, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_64stack:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]
+; X86-AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_64stack:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; X64-SSE-NEXT:    divsd -{{[0-9]+}}(%rsp), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_64stack:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; X64-AVX-NEXT:    vdivsd -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    retq
+  %ptr = alloca i64, align 8
+  %load = load atomic i64, ptr %ptr acquire, align 8
+  %bc0 = bitcast i64 %load to double
+  %fdiv = fdiv double 1.000000e+00, %bc0
+  %bc1 = bitcast double %fdiv to i64
+  store atomic i64 %bc1, ptr %ptr release, align 8
+  ret void
+}
+
+define dso_local void @fdiv_array(ptr %arg, double %arg1, i64 %arg2) nounwind {
+; X86-NOSSE-LABEL: fdiv_array:
+; X86-NOSSE:       # %bb.0: # %bb
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    andl $-8, %esp
+; X86-NOSSE-NEXT:    subl $40, %esp
+; X86-NOSSE-NEXT:    movl 20(%ebp), %eax
+; X86-NOSSE-NEXT:    movl 8(%ebp), %ecx
+; X86-NOSSE-NEXT:    fildll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fdivl 12(%ebp)
+; X86-NOSSE-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOSSE-NEXT:    movl %edx, (%esp)
+; X86-NOSSE-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    fildll (%esp)
+; X86-NOSSE-NEXT:    fistpll (%ecx,%eax,8)
+; X86-NOSSE-NEXT:    leal -4(%ebp), %esp
+; X86-NOSSE-NEXT:    popl %esi
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: fdiv_array:
+; X86-SSE1:       # %bb.0: # %bb
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    movl %esp, %ebp
+; X86-SSE1-NEXT:    andl $-8, %esp
+; X86-SSE1-NEXT:    subl $16, %esp
+; X86-SSE1-NEXT:    movl 20(%ebp), %eax
+; X86-SSE1-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
+; X86-SSE1-NEXT:    movss %xmm1, (%esp)
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    fldl (%esp)
+; X86-SSE1-NEXT:    fdivl 12(%ebp)
+; X86-SSE1-NEXT:    fstpl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE1-NEXT:    movl %ebp, %esp
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fdiv_array:
+; X86-SSE2:       # %bb.0: # %bb
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 20(%ebp), %eax
+; X86-SSE2-NEXT:    movl 8(%ebp), %ecx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    divsd 12(%ebp), %xmm0
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: fdiv_array:
+; X86-AVX:       # %bb.0: # %bb
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 20(%ebp), %eax
+; X86-AVX-NEXT:    movl 8(%ebp), %ecx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vdivsd 12(%ebp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%ecx,%eax,8)
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: fdiv_array:
+; X64-SSE:       # %bb.0: # %bb
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-SSE-NEXT:    divsd %xmm0, %xmm1
+; X64-SSE-NEXT:    movsd %xmm1, (%rdi,%rsi,8)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fdiv_array:
+; X64-AVX:       # %bb.0: # %bb
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi,%rsi,8)
+; X64-AVX-NEXT:    retq
+bb:
+  %tmp4 = getelementptr inbounds i64, ptr %arg, i64 %arg2
+  %tmp6 = load atomic i64, ptr %tmp4 monotonic, align 8
+  %tmp7 = bitcast i64 %tmp6 to double
+  %tmp8 = fdiv double %tmp7, %arg1
+  %tmp9 = bitcast double %tmp8 to i64
+  store atomic i64 %tmp9, ptr %tmp4 monotonic, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
index 47ac8848a437d8eea3b05869c23f7e4e2532625b..ac65a1112be439422f3da7d9eec50642702a030c 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -1367,7 +1367,7 @@ define void @bcast_unfold_fdiv_v16f32(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v16f32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB42_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -1386,7 +1386,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp
   %tmp4 = load <16 x float>, ptr %tmp2, align 4
-  %tmp5 = fdiv <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp5 = fdiv <16 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   store <16 x float> %tmp5, ptr %tmp2, align 4
   %tmp7 = add i64 %tmp, 16
   %tmp8 = icmp eq i64 %tmp7, 1024
@@ -1400,7 +1400,7 @@ define void @bcast_unfold_fdiv_v8f32(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v8f32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB43_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -1419,7 +1419,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp
   %tmp4 = load <8 x float>, ptr %tmp2, align 4
-  %tmp5 = fdiv <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp5 = fdiv <8 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   store <8 x float> %tmp5, ptr %tmp2, align 4
   %tmp7 = add i64 %tmp, 8
   %tmp8 = icmp eq i64 %tmp7, 1024
@@ -1433,7 +1433,7 @@ define void @bcast_unfold_fdiv_v4f32(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v4f32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB44_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -1451,7 +1451,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp
   %tmp4 = load <4 x float>, ptr %tmp2, align 4
-  %tmp5 = fdiv <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %tmp5 = fdiv <4 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   store <4 x float> %tmp5, ptr %tmp2, align 4
   %tmp7 = add i64 %tmp, 4
   %tmp8 = icmp eq i64 %tmp7, 1024
@@ -1465,7 +1465,7 @@ define void @bcast_unfold_fdiv_v8f64(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v8f64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB45_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -1484,7 +1484,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp
   %tmp4 = load <8 x double>, ptr %tmp2, align 8
-  %tmp5 = fdiv <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp5 = fdiv <8 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
   store <8 x double> %tmp5, ptr %tmp2, align 8
   %tmp7 = add i64 %tmp, 8
   %tmp8 = icmp eq i64 %tmp7, 1024
@@ -1498,7 +1498,7 @@ define void @bcast_unfold_fdiv_v4f64(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v4f64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB46_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -1517,7 +1517,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp
   %tmp4 = load <4 x double>, ptr %tmp2, align 8
-  %tmp5 = fdiv <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+  %tmp5 = fdiv <4 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
   store <4 x double> %tmp5, ptr %tmp2, align 8
   %tmp7 = add i64 %tmp, 4
   %tmp8 = icmp eq i64 %tmp7, 1024
@@ -1531,7 +1531,7 @@ define void @bcast_unfold_fdiv_v2f64(ptr nocapture %arg) {
 ; CHECK-LABEL: bcast_unfold_fdiv_v2f64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [3.0E+0,3.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB47_1: # %bb1
@@ -1550,7 +1550,7 @@ bb1:                                              ; preds = %bb1, %bb
   %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]
   %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp
   %tmp4 = load <2 x double>, ptr %tmp2, align 8
-  %tmp5 = fdiv <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>
+  %tmp5 = fdiv <2 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00>
   store <2 x double> %tmp5, ptr %tmp2, align 8
   %tmp7 = add i64 %tmp, 2
   %tmp8 = icmp eq i64 %tmp7, 1024
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
index 1782e52528700d3d00bca8078f311af861fd0e44..55b86cadfe30ee597560c18c6c98da5691c1ae4e 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
@@ -28,35 +28,17 @@ define half @test_fminimum(half %x, half %y) {
 define <8 x half> @test_fminimum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
 ; CHECK-LABEL: test_fminimum_scalarize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vminsh %xmm2, %xmm3, %xmm2
-; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; CHECK-NEXT:    vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,0]
-; CHECK-NEXT:    vshufpd {{.*#+}} xmm5 = xmm0[1,0]
-; CHECK-NEXT:    vminsh %xmm4, %xmm5, %xmm4
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT:    vpsrlq $48, %xmm1, %xmm3
-; CHECK-NEXT:    vpsrlq $48, %xmm0, %xmm4
-; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vminsh %xmm4, %xmm5, %xmm4
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT:    vminsh %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vpsrld $16, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrld $16, %xmm0, %xmm0
-; CHECK-NEXT:    vminsh %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    vcmpltph %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm2 {%k1}
+; CHECK-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; CHECK-NEXT:    vpcmpeqw %xmm3, %xmm0, %k1
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm2, %xmm0 {%k1}
+; CHECK-NEXT:    vpcmpeqw %xmm3, %xmm1, %k1
+; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1}
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vcmpeqph %xmm1, %xmm2, %k1
+; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %r = call <8 x half> @llvm.minimum.v8f16(<8 x half> %x, <8 x half> %y)
   ret <8 x half> %r
@@ -134,35 +116,16 @@ define half @test_fmaximum(half %x, half %y) {
 define <8 x half> @test_fmaximum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
 ; CHECK-LABEL: test_fmaximum_scalarize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vmaxsh %xmm2, %xmm3, %xmm2
-; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; CHECK-NEXT:    vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,0]
-; CHECK-NEXT:    vshufpd {{.*#+}} xmm5 = xmm0[1,0]
-; CHECK-NEXT:    vmaxsh %xmm4, %xmm5, %xmm4
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT:    vpsrlq $48, %xmm1, %xmm3
-; CHECK-NEXT:    vpsrlq $48, %xmm0, %xmm4
-; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
-; CHECK-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vmaxsh %xmm4, %xmm5, %xmm4
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vpsrld $16, %xmm1, %xmm1
-; CHECK-NEXT:    vpsrld $16, %xmm0, %xmm0
-; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    vcmpltph %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm2 {%k1}
+; CHECK-NEXT:    vptestnmw %xmm0, %xmm0, %k1
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm2, %xmm0 {%k1}
+; CHECK-NEXT:    vptestnmw %xmm1, %xmm1, %k1
+; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1}
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vcmpeqph %xmm1, %xmm2, %k1
+; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %r = call <8 x half> @llvm.maximum.v8f16(<8 x half> %x, <8 x half> %y)
   ret <8 x half> %r
diff --git a/llvm/test/CodeGen/X86/bug47278-eflags-error.mir b/llvm/test/CodeGen/X86/bug47278-eflags-error.mir
index e4e68451850b7d12b8da740ddfed3a23a77190f7..65d368ee224deedede39b9df61c3b032b7c435e1 100644
--- a/llvm/test/CodeGen/X86/bug47278-eflags-error.mir
+++ b/llvm/test/CodeGen/X86/bug47278-eflags-error.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -passes=regallocfast -o - %s | FileCheck %s
 
 # Test for correct management of allocatable and non-allocatable
 # live-ins in fastregalloc
diff --git a/llvm/test/CodeGen/X86/bug47278.mir b/llvm/test/CodeGen/X86/bug47278.mir
index 0b01c9de19c0b7921d8eb74f37af537cfedc7b1a..ea7bb096169819e60a691161e6f582cb18d71e57 100644
--- a/llvm/test/CodeGen/X86/bug47278.mir
+++ b/llvm/test/CodeGen/X86/bug47278.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -run-pass=regallocfast -o -  %s | FileCheck %s
+# RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -passes=regallocfast -o -  %s | FileCheck %s
 
 # Make sure this case doesn't assert or try to assign $ecx to %1 on
 # SHRD32rrCL
diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
index 6417538b269ccdae56d3b6f3ab5a6e552b929972..6263a3f09b0d3f790b46468feeb3ce1dbf8e333f 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
+++ b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -run-pass=regallocfast -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -passes=regallocfast -verify-machineinstrs %s -o - | FileCheck %s
 --- |
   ; ModuleID = 'x.c'
   source_filename = "x.c"
diff --git a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
index 33a7ec9bfc79445215b5d2ba8c509cd3ac1409fe..ba09ba8b6402bead814442d764e0d0a5e0d33d10 100644
--- a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
+++ b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
@@ -14,7 +14,7 @@
 define double @unsafe_fp_math_default0(double %x) {
 ; SAFE:      divsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 2.0
+  %div = fdiv double %x, 3.0
   ret double %div
 }
 
@@ -22,7 +22,7 @@ define double @unsafe_fp_math_default0(double %x) {
 define double @unsafe_fp_math_off(double %x) #0 {
 ; SAFE:      divsd
 ; UNSAFE:    divsd
-  %div = fdiv double %x, 2.0
+  %div = fdiv double %x, 3.0
   ret double %div
 }
 
@@ -31,7 +31,7 @@ define double @unsafe_fp_math_default1(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
 ; SAFE:      divsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 2.0
+  %div = fdiv double %x, 3.0
   ret double %div
 }
 
@@ -39,7 +39,7 @@ define double @unsafe_fp_math_default1(double %x) {
 define double @unsafe_fp_math_on(double %x) #1 {
 ; SAFE:      mulsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 2.0
+  %div = fdiv double %x, 3.0
   ret double %div
 }
 
@@ -48,7 +48,7 @@ define double @unsafe_fp_math_default2(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
 ; SAFE:      divsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 2.0
+  %div = fdiv double %x, 3.0
   ret double %div
 }
 
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 33cc8e96f663f5a849caa789b1bfd12024541dd8..1c303de55c95db73ed08b9d38c019a7f75e21cef 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -177,56 +177,55 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $152, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    subl $156, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    sarl $31, %eax
-; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    sarl $31, %ebx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    xorl %ecx, %esi
-; X86-NEXT:    movl %esi, %ebp
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, %edi
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %eax, %esi
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    subl %eax, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %ecx
+; X86-NEXT:    xorl %eax, %esi
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    xorl %eax, %edx
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    xorl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %eax, %ecx
+; X86-NEXT:    subl %eax, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %eax, %esi
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %edi
 ; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %ebp
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    xorl %edx, %esi
-; X86-NEXT:    movl %ebx, %edx
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl %ebx, %edi
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %ebx, %ebp
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    subl %ebx, %ebp
-; X86-NEXT:    sbbl %ebx, %edi
-; X86-NEXT:    sbbl %ebx, %edx
-; X86-NEXT:    sbbl %ebx, %esi
-; X86-NEXT:    xorl %eax, %ebx
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    xorl %edx, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    xorl %edx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %edx, %edi
+; X86-NEXT:    subl %edx, %edi
+; X86-NEXT:    sbbl %edx, %ebp
+; X86-NEXT:    sbbl %edx, %ebx
+; X86-NEXT:    sbbl %edx, %esi
+; X86-NEXT:    xorl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebp, %eax
 ; X86-NEXT:    orl %esi, %eax
-; X86-NEXT:    movl %ebp, %ecx
-; X86-NEXT:    orl %edx, %ecx
-; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    orl %ebx, %ecx
 ; X86-NEXT:    orl %eax, %ecx
 ; X86-NEXT:    sete %cl
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    orl (%esp), %eax # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    orl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    orl %eax, %edx
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    orb %cl, %al
@@ -238,76 +237,78 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %esi, %esi
 ; X86-NEXT:    cmovnel %edx, %ecx
-; X86-NEXT:    bsrl %edi, %edx
+; X86-NEXT:    bsrl %ebp, %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    bsrl %ebp, %ebp
-; X86-NEXT:    xorl $31, %ebp
-; X86-NEXT:    addl $32, %ebp
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    testl %edi, %edi
-; X86-NEXT:    cmovnel %edx, %ebp
-; X86-NEXT:    addl $64, %ebp
+; X86-NEXT:    bsrl %edi, %edi
+; X86-NEXT:    xorl $31, %edi
+; X86-NEXT:    addl $32, %edi
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    cmovnel %edx, %edi
+; X86-NEXT:    addl $64, %edi
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, %edx
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %esi, %ebx
-; X86-NEXT:    cmovnel %ecx, %ebp
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    orl %esi, %edx
+; X86-NEXT:    cmovnel %ecx, %edi
+; X86-NEXT:    movl (%esp), %ebx # 4-byte Reload
 ; X86-NEXT:    bsrl %ebx, %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X86-NEXT:    bsrl %eax, %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT:    bsrl %ebp, %ecx
 ; X86-NEXT:    xorl $31, %ecx
 ; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    bsrl %edi, %esi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    bsrl %eax, %esi
 ; X86-NEXT:    xorl $31, %esi
 ; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    addl $32, %edx
-; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    testl %eax, %eax
 ; X86-NEXT:    cmovnel %esi, %edx
 ; X86-NEXT:    addl $64, %edx
-; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ebp, %esi
 ; X86-NEXT:    orl %ebx, %esi
 ; X86-NEXT:    cmovnel %ecx, %edx
-; X86-NEXT:    xorl %esi, %esi
-; X86-NEXT:    subl %edx, %ebp
-; X86-NEXT:    movl $0, %edi
-; X86-NEXT:    sbbl %edi, %edi
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    subl %edx, %edi
 ; X86-NEXT:    movl $0, %edx
 ; X86-NEXT:    sbbl %edx, %edx
 ; X86-NEXT:    movl $0, %eax
 ; X86-NEXT:    sbbl %eax, %eax
+; X86-NEXT:    movl $0, %esi
+; X86-NEXT:    sbbl %esi, %esi
 ; X86-NEXT:    movl $127, %ecx
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    cmpl %ebp, %ecx
-; X86-NEXT:    movl %edx, %ebp
-; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    sbbl %edi, %ecx
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    cmpl %edi, %ecx
 ; X86-NEXT:    movl $0, %ecx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %edx, %ecx
 ; X86-NEXT:    movl $0, %ecx
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ecx
+; X86-NEXT:    movl $0, %ecx
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %esi, %ecx
 ; X86-NEXT:    setb %cl
 ; X86-NEXT:    orb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
-; X86-NEXT:    movl %ebx, %edx
-; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    movl (%esp), %ebx # 4-byte Reload
-; X86-NEXT:    cmovnel %esi, %ebx
+; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    cmovnel %ebx, %edx
+; X86-NEXT:    cmovnel %ebx, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    cmovnel %esi, %eax
-; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT:    cmovnel %ebx, %eax
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT:    movl %ebx, %esi
 ; X86-NEXT:    jne .LBB4_8
 ; X86-NEXT:  # %bb.1: # %_udiv-special-cases
-; X86-NEXT:    movl %edi, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    xorl $127, %edi
-; X86-NEXT:    orl %ebp, %edi
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    movl %ebx, %ecx
 ; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    je .LBB4_8
@@ -316,10 +317,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
-; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
@@ -333,27 +334,27 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    andb $15, %al
 ; X86-NEXT:    negb %al
 ; X86-NEXT:    movsbl %al, %edi
-; X86-NEXT:    movl 144(%esp,%edi), %edx
-; X86-NEXT:    movl 148(%esp,%edi), %esi
+; X86-NEXT:    movl 148(%esp,%edi), %edx
+; X86-NEXT:    movl 152(%esp,%edi), %esi
 ; X86-NEXT:    movb %ch, %cl
 ; X86-NEXT:    shldl %cl, %edx, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    notb %cl
-; X86-NEXT:    movl 140(%esp,%edi), %eax
-; X86-NEXT:    movl %eax, %ebx
-; X86-NEXT:    shrl %ebx
-; X86-NEXT:    shrl %cl, %ebx
-; X86-NEXT:    orl %edx, %ebx
-; X86-NEXT:    movl 136(%esp,%edi), %edx
+; X86-NEXT:    movl 144(%esp,%edi), %eax
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    shrl %ebp
+; X86-NEXT:    shrl %cl, %ebp
+; X86-NEXT:    orl %edx, %ebp
+; X86-NEXT:    movl 140(%esp,%edi), %edx
 ; X86-NEXT:    movb %ch, %cl
 ; X86-NEXT:    shldl %cl, %edx, %eax
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    addl $1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    adcl $0, %ecx
-; X86-NEXT:    adcl $0, %ebp
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT:    adcl $0, %edi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    adcl $0, %edx
 ; X86-NEXT:    jae .LBB4_3
@@ -363,38 +364,37 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    jmp .LBB4_7
 ; X86-NEXT:  .LBB4_3: # %udiv-preheader
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl (%esp), %esi # 4-byte Reload
-; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movb %dl, %ch
 ; X86-NEXT:    andb $7, %ch
 ; X86-NEXT:    movb %dl, %cl
 ; X86-NEXT:    shrb $3, %cl
 ; X86-NEXT:    andb $15, %cl
 ; X86-NEXT:    movzbl %cl, %edx
-; X86-NEXT:    movl 100(%esp,%edx), %esi
-; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X86-NEXT:    movl 96(%esp,%edx), %edi
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl 104(%esp,%edx), %ebx
+; X86-NEXT:    movl 100(%esp,%edx), %edi
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edi, %ebp
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrdl %cl, %esi, %ebp
-; X86-NEXT:    movl 88(%esp,%edx), %ebx
+; X86-NEXT:    shrdl %cl, %ebx, %ebp
 ; X86-NEXT:    movl 92(%esp,%edx), %esi
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl 96(%esp,%edx), %esi
 ; X86-NEXT:    movl %esi, %edx
 ; X86-NEXT:    shrl %cl, %edx
 ; X86-NEXT:    notb %cl
@@ -403,9 +403,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    orl %edx, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; X86-NEXT:    shrdl %cl, %esi, %ebx
+; X86-NEXT:    shrl %cl, %ebx
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    addl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -415,7 +415,6 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -424,26 +423,26 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB4_4: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebp, %esi
-; X86-NEXT:    shldl $1, %ebp, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    shldl $1, %ebp, %ebx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebp, %esi
+; X86-NEXT:    shldl $1, %ebp, (%esp) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %edx, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    shldl $1, %edi, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %ecx, %edi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    orl %ebx, %edi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    orl %esi, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    shldl $1, %eax, %ecx
-; X86-NEXT:    orl %ebx, %ecx
+; X86-NEXT:    orl %esi, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %ecx, %eax
-; X86-NEXT:    orl %ebx, %eax
+; X86-NEXT:    orl %esi, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    addl %ecx, %ecx
 ; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
@@ -452,15 +451,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    sbbl %ebp, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl %esi, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    sarl $31, %ecx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl $1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ecx, %ebx
-; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X86-NEXT:    movl %ecx, %eax
@@ -468,21 +467,22 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    subl %ecx, %edx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    sbbl %eax, %ebp
 ; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %edi, %esi
-; X86-NEXT:    movl %esi, %ebp
+; X86-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X86-NEXT:    sbbl %edi, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    sbbl %ebx, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    sbbl %esi, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    addl $-1, %ecx
-; X86-NEXT:    adcl $-1, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT:    adcl $-1, %edi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ebx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %esi
-; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    orl %esi, %edi
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -491,15 +491,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    jne .LBB4_4
 ; X86-NEXT:  # %bb.5:
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:  .LBB4_7: # %udiv-loop-exit
-; X86-NEXT:    shldl $1, %ebx, %edx
+; X86-NEXT:    shldl $1, %ebp, %edx
 ; X86-NEXT:    orl %ecx, %edx
-; X86-NEXT:    shldl $1, %eax, %ebx
-; X86-NEXT:    orl %ecx, %ebx
+; X86-NEXT:    shldl $1, %eax, %ebp
+; X86-NEXT:    orl %ecx, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    shldl $1, %esi, %eax
 ; X86-NEXT:    orl %ecx, %eax
@@ -508,36 +508,35 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:  .LBB4_8: # %udiv-end
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    xorl %ecx, %edx
-; X86-NEXT:    xorl %ecx, %ebx
+; X86-NEXT:    xorl %ecx, %ebp
 ; X86-NEXT:    xorl %ecx, %eax
 ; X86-NEXT:    xorl %ecx, %esi
 ; X86-NEXT:    subl %ecx, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %ecx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %ecx, %ebx
+; X86-NEXT:    sbbl %ecx, %ebp
 ; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %esi, (%ecx)
 ; X86-NEXT:    movl %eax, 4(%ecx)
-; X86-NEXT:    movl %ebx, 8(%ecx)
+; X86-NEXT:    movl %ebp, 8(%ecx)
 ; X86-NEXT:    movl %edx, 12(%ecx)
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    movl %ebp, %edi
 ; X86-NEXT:    mull %ecx
 ; X86-NEXT:    movl %edx, %ebx
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %eax, %ebp
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    mull %ecx
 ; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
 ; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    addl %ebp, %ecx
 ; X86-NEXT:    adcl $0, %ebx
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    mull %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    mull %ebp
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    adcl %ebx, %edx
@@ -545,7 +544,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    setb %cl
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    mull %ebp
 ; X86-NEXT:    addl %ebx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movzbl %cl, %eax
@@ -556,35 +555,36 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    imull %eax, %ecx
 ; X86-NEXT:    mull %edi
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    imull {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    imull %ebp, %edi
 ; X86-NEXT:    addl %edx, %edi
 ; X86-NEXT:    addl %ecx, %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    imull %esi, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    imull %edx, %ebp
+; X86-NEXT:    imull %edx, %esi
 ; X86-NEXT:    mull %edx
-; X86-NEXT:    addl %edx, %ebp
-; X86-NEXT:    addl %ecx, %ebp
+; X86-NEXT:    addl %edx, %esi
+; X86-NEXT:    addl %ecx, %esi
 ; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    adcl %edi, %ebp
+; X86-NEXT:    adcl %edi, %esi
 ; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    subl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    sbbl %eax, %esi
+; X86-NEXT:    subl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    sbbl %eax, %ebx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    sbbl %ebp, %edi
+; X86-NEXT:    sbbl %esi, %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %edx, (%eax)
-; X86-NEXT:    movl %ecx, 4(%eax)
-; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %ebx, 8(%eax)
 ; X86-NEXT:    movl %edi, 12(%eax)
-; X86-NEXT:    addl $152, %esp
+; X86-NEXT:    addl $156, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
diff --git a/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir b/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir
index d60632a98fb00e1d3485585766c246ca2df54fab..1872e703d6cd454570b635fcfd737c6766ad9cea 100644
--- a/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir
+++ b/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s
 
 ...
 ---
diff --git a/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir b/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir
index 8a6311eabacadb951e9a02576ec7bcbc7f3d2549..18de9c11439c94938e36705cf50a589423c9cbd7 100644
--- a/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir
+++ b/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s
 
 # If the tied use is undef value, fastregalloc should free the def register.
 # There is no reload needed for the undef value.
diff --git a/llvm/test/CodeGen/X86/llvm.tan.ll b/llvm/test/CodeGen/X86/llvm.tan.ll
new file mode 100644
index 0000000000000000000000000000000000000000..24b30038687f2b423a805b0907df101139894faa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/llvm.tan.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define half @use_tanf16(half %a) nounwind {
+; CHECK-LABEL: use_tanf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq __extendhfsf2@PLT
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    callq __truncsfhf2@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+  %x = call half @llvm.tan.f16(half %a)
+  ret half %x
+}
+
+define float @use_tanf32(float %a) nounwind {
+; CHECK-LABEL: use_tanf32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp tanf@PLT # TAILCALL
+  %x = call float @llvm.tan.f32(float %a)
+  ret float %x
+}
+
+define double @use_tanf64(double %a) nounwind {
+; CHECK-LABEL: use_tanf64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp tan@PLT # TAILCALL
+  %x = call double @llvm.tan.f64(double %a)
+  ret double %x
+}
+
+define x86_fp80 @use_tanf80(x86_fp80 %a) nounwind {
+; CHECK-LABEL: use_tanf80:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    fldt 32(%rsp)
+; CHECK-NEXT:    fstpt (%rsp)
+; CHECK-NEXT:    callq tanl@PLT
+; CHECK-NEXT:    addq  $24, %rsp
+; CHECK-NEXT:    retq
+  %x = call x86_fp80 @llvm.tan.f80(x86_fp80 %a)
+  ret x86_fp80 %x
+}
+
+define fp128 @use_tanfp128(fp128 %a) nounwind {
+; CHECK-LABEL: use_tanfp128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    jmp  tanf128@PLT # TAILCALL
+  %x = call fp128 @llvm.tan.f128(fp128 %a)
+  ret fp128 %x
+}
+
+define ppc_fp128 @use_tanppc_fp128(ppc_fp128 %a) nounwind {
+; CHECK-LABEL: use_tanppc_fp128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq tanl@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+  %x = call ppc_fp128 @llvm.tan.ppcf128(ppc_fp128 %a)
+  ret ppc_fp128 %x
+}
+
+declare half @llvm.tan.f16(half)
+declare float @llvm.tan.f32(float)
+declare double @llvm.tan.f64(double)
+declare x86_fp80 @llvm.tan.f80(x86_fp80)
+declare fp128 @llvm.tan.f128(fp128)
+declare ppc_fp128 @llvm.tan.ppcf128(ppc_fp128)
diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll
index ace78b38d53edbb500a29bdd63ca82f4a7aa1bf0..fbc363f77ec42c9882378b1e9d638ef87bfc7680 100644
--- a/llvm/test/CodeGen/X86/pr38539.ll
+++ b/llvm/test/CodeGen/X86/pr38539.ll
@@ -36,9 +36,9 @@ define void @f() nounwind {
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    sarl $30, %ecx
 ; X86-NEXT:    sarl $31, %eax
-; X86-NEXT:    shrdl $1, %eax, %ecx
 ; X86-NEXT:    xorl %eax, %edx
 ; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    shrdl $1, %eax, %ecx
 ; X86-NEXT:    xorl %ecx, %esi
 ; X86-NEXT:    subl %ecx, %esi
 ; X86-NEXT:    sbbl %eax, %edi
diff --git a/llvm/test/CodeGen/X86/pr94824.ll b/llvm/test/CodeGen/X86/pr94824.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7744d00acf3d481fc5b19e366ce53e55148e53ea
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr94824.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define i16 @pr94824(i8 %x1) {
+; CHECK-LABEL: pr94824:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    orl $256, %edi # imm = 0x100
+; CHECK-NEXT:    rep bsfl %edi, %ecx
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+entry:
+  %cttz = call i8 @llvm.cttz.i8(i8 %x1, i1 false)
+  %ext = zext i8 %cttz to i16
+  %shl = shl i16 1, %ext
+  ret i16 %shl
+}
diff --git a/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir b/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir
index 6a2eae9dbbc4c2ba71b50ec0b82450c96a42c34c..143b6864b1191bd54fc8bfae7c2566968900f642 100644
--- a/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir
+++ b/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=x86_64-grtev4-linux-gnu -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-grtev4-linux-gnu -passes=regallocfast -o - %s | FileCheck %s
 
 # Bug 41973. Make sure %12 is detected as live out of %bb.0, even
 # though the use is allocated before the def block %bb.3. Previously
diff --git a/llvm/test/CodeGen/X86/select-phi-s16-fp.ll b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..083f418b6752eb28b1604474b4d896191e20f780
--- /dev/null
+++ b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mcpu=generic -mtriple=x86_64-apple-darwin %s -o - | FileCheck %s
+
+; For all these tests we disable optimizations through function attributes
+; because the code we are exercising here needs phis and we want to keep the
+; IR small.
+
+; This code used to crash in SDISel because bf16 was promoted to f32 through
+; a `f32 = vector_extract_elt <1 x bf16>, i32 0`, which is illegal.
+; The invalid SDNode and thus, the crash was only exposed by the constant
+; folding.
+define void @phi_vec1bf16_to_f32_with_const_folding(ptr %dst) #0 {
+; CHECK-LABEL: phi_vec1bf16_to_f32_with_const_folding:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    jmp LBB0_1
+; CHECK-NEXT:  LBB0_1: ## %bb
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    callq ___truncsfbf2
+; CHECK-NEXT:    pextrw $0, %xmm0, %eax
+; CHECK-NEXT:    movw %ax, 2(%rbx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+entry:
+  br label %bb
+
+bb:
+  %phi = phi <1 x bfloat> [ zeroinitializer, %entry ]
+  %res = shufflevector <1 x bfloat> poison, <1 x bfloat> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x bfloat> %res, ptr %dst
+  ret void
+}
+; Same as phi_vec1bf16_to_f32_with_const_folding but without the constant
+; folding.
+; This test exercises the same invalid SDNode, but it happened to work by
+; accident before. Here we make sure the fix also work as expected in the
+; non-constant folding case.
+define void @phi_vec1bf16_to_f32(ptr %src, ptr %dst) #0 {
+; CHECK-LABEL: phi_vec1bf16_to_f32:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    jmp LBB1_1
+; CHECK-NEXT:  LBB1_1: ## %bb
+; CHECK-NEXT:    callq ___truncsfbf2
+; CHECK-NEXT:    pextrw $0, %xmm0, %eax
+; CHECK-NEXT:    movw %ax, 2(%rbx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+entry:
+  %input = load <1 x bfloat>, ptr %src
+  br label %bb
+
+bb:
+  %phi = phi <1 x bfloat> [ %input, %entry ]
+  %res = shufflevector <1 x bfloat> poison, <1 x bfloat> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x bfloat> %res, ptr %dst
+  ret void
+}
+
+
+; Half type is legal on x86 so nothing special here, but it
+; doesn't hurt to be thorough.
+define void @phi_vec1half_with_const_folding(ptr %dst) #0 {
+; CHECK-LABEL: phi_vec1half_with_const_folding:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    jmp LBB2_1
+; CHECK-NEXT:  LBB2_1: ## %bb
+; CHECK-NEXT:    pextrw $0, %xmm0, %eax
+; CHECK-NEXT:    movw %ax, 2(%rdi)
+; CHECK-NEXT:    retq
+entry:
+  br label %bb
+
+bb:
+  %phi = phi <1 x half> [ zeroinitializer, %entry ]
+  %res = shufflevector <1 x half> poison, <1 x half> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x half> %res, ptr %dst
+  ret void
+}
+
+; Half type is legal on x86 so nothing special here, but it
+; doesn't hurt to be thorough.
+; Same as phi_vec1half_with_constant_folding but without the constant folding.
+define void @phi_vec1half(ptr %src, ptr %dst) #0 {
+; CHECK-LABEL: phi_vec1half:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pinsrw $0, (%rdi), %xmm0
+; CHECK-NEXT:    jmp LBB3_1
+; CHECK-NEXT:  LBB3_1: ## %bb
+; CHECK-NEXT:    pextrw $0, %xmm0, %eax
+; CHECK-NEXT:    movw %ax, 2(%rsi)
+; CHECK-NEXT:    retq
+entry:
+  %input = load <1 x half>, ptr %src
+  br label %bb
+
+bb:
+  %phi = phi <1 x half> [ %input, %entry ]
+  %res = shufflevector <1 x half> poison, <1 x half> %phi, <2 x i32> <i32 0, i32 1>
+  store <2 x half> %res, ptr %dst
+  ret void
+}
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir
index 8bac14018a7dd35973296b0201a5051f88f3aa7e..f683e9ebec81d74065572de582dcce27e7a186e9 100644
--- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s
 
 # Check that fastregalloc does not displace register assigned to tied def when
 # RegMask operand is present. STATEPOINT is an example of such instruction.
diff --git a/llvm/test/CodeGen/X86/vec-libcalls.ll b/llvm/test/CodeGen/X86/vec-libcalls.ll
index 3a1315446d7a2c3af09acc6b6f08832683b91dbe..6857101d3d75bbbc469aa99aecd703d9e33f72fa 100644
--- a/llvm/test/CodeGen/X86/vec-libcalls.ll
+++ b/llvm/test/CodeGen/X86/vec-libcalls.ll
@@ -17,6 +17,14 @@ declare <5 x float> @llvm.sin.v5f32(<5 x float>)
 declare <6 x float> @llvm.sin.v6f32(<6 x float>)
 declare <3 x double> @llvm.sin.v3f64(<3 x double>)
 
+declare <1 x float> @llvm.tan.v1f32(<1 x float>)
+declare <2 x float> @llvm.tan.v2f32(<2 x float>)
+declare <3 x float> @llvm.tan.v3f32(<3 x float>)
+declare <4 x float> @llvm.tan.v4f32(<4 x float>)
+declare <5 x float> @llvm.tan.v5f32(<5 x float>)
+declare <6 x float> @llvm.tan.v6f32(<6 x float>)
+declare <3 x double> @llvm.tan.v3f64(<3 x double>)
+
 ; Verify that all of the potential libcall candidates are handled.
 ; Some of these have custom lowering, so those cases won't have
 ; libcalls.
@@ -230,6 +238,200 @@ define <3 x double> @sin_v3f64(<3 x double> %x) nounwind {
   ret <3 x double> %r
 }
 
+define <1 x float> @tan_v1f32(<1 x float> %x) nounwind {
+; CHECK-LABEL: tan_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+  %r = call <1 x float> @llvm.tan.v1f32(<1 x float> %x)
+  ret <1 x float> %r
+}
+
+define <2 x float> @tan_v2f32(<2 x float> %x) nounwind {
+; CHECK-LABEL: tan_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+  %r = call <2 x float> @llvm.tan.v2f32(<2 x float> %x)
+  ret <2 x float> %r
+}
+
+define <3 x float> @tan_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: tan_v3f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+  %r = call <3 x float> @llvm.tan.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
+define <4 x float> @tan_v4f32(<4 x float> %x) nounwind {
+; CHECK-LABEL: tan_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+  %r = call <4 x float> @llvm.tan.v4f32(<4 x float> %x)
+  ret <4 x float> %r
+}
+
+define <5 x float> @tan_v5f32(<5 x float> %x) nounwind {
+; CHECK-LABEL: tan_v5f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $72, %rsp
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; CHECK-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    addq $72, %rsp
+; CHECK-NEXT:    retq
+  %r = call <5 x float> @llvm.tan.v5f32(<5 x float> %x)
+  ret <5 x float> %r
+}
+
+define <6 x float> @tan_v6f32(<6 x float> %x) nounwind {
+; CHECK-LABEL: tan_v6f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $72, %rsp
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
+; CHECK-NEXT:    callq tanf@PLT
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; CHECK-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT:    addq $72, %rsp
+; CHECK-NEXT:    retq
+  %r = call <6 x float> @llvm.tan.v6f32(<6 x float> %x)
+  ret <6 x float> %r
+}
+
+define <3 x double> @tan_v3f64(<3 x double> %x) nounwind {
+; CHECK-LABEL: tan_v3f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $72, %rsp
+; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tan@PLT
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq tan@PLT
+; CHECK-NEXT:    vmovapd (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
+; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq tan@PLT
+; CHECK-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    addq $72, %rsp
+; CHECK-NEXT:    retq
+  %r = call <3 x double> @llvm.tan.v3f64(<3 x double> %x)
+  ret <3 x double> %r
+}
+
 define <2 x float> @fabs_v2f32(<2 x float> %x) nounwind {
 ; CHECK-LABEL: fabs_v2f32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir b/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir
index 6cdac85f96fe9ba9f28e2d1ed1ed6cd4ebd02673..5222b3d9fdb859a445a4800691af69e076aa208f 100644
--- a/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir
+++ b/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir
@@ -1,4 +1,5 @@
 # RUN: llc -o - -mtriple=x86_64-- -run-pass=regallocfast %s | FileCheck %s
+# RUN: llc -o - -mtriple=x86_64-- -passes=regallocfast %s | FileCheck %s
 # Fast regalloc used to not collect physical register definitions
 # before walking and assigning the virtual definition.
 # Therefore it was possible for a virtual definition to end up
diff --git a/llvm/test/CodeGen/X86/vshift-6.ll b/llvm/test/CodeGen/X86/vshift-6.ll
index 21c8e8d3ee5d214a38c7bdb34e8414cdc37b1e4d..912ff750d9e91f2dd468d3ed83fedc31fe97a476 100644
--- a/llvm/test/CodeGen/X86/vshift-6.ll
+++ b/llvm/test/CodeGen/X86/vshift-6.ll
@@ -32,9 +32,9 @@ define <16 x i8> @do_not_crash(ptr, ptr, ptr, i32, i64, i8) {
 ; X86-NEXT:    movb %al, (%ecx)
 ; X86-NEXT:    movd %eax, %xmm1
 ; X86-NEXT:    psllq $56, %xmm1
-; X86-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-NEXT:    pcmpeqd %xmm3, %xmm3
 ; X86-NEXT:    psllw $5, %xmm1
+; X86-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-NEXT:    pxor %xmm2, %xmm2
 ; X86-NEXT:    pxor %xmm0, %xmm0
 ; X86-NEXT:    pcmpgtb %xmm1, %xmm0
@@ -64,9 +64,9 @@ define <16 x i8> @do_not_crash(ptr, ptr, ptr, i32, i64, i8) {
 ; X64-NEXT:    movb %r9b, (%rdi)
 ; X64-NEXT:    movd %r9d, %xmm1
 ; X64-NEXT:    psllq $56, %xmm1
-; X64-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; X64-NEXT:    pcmpeqd %xmm2, %xmm2
 ; X64-NEXT:    psllw $5, %xmm1
+; X64-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; X64-NEXT:    pxor %xmm3, %xmm3
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtb %xmm1, %xmm0
diff --git a/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll b/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll
new file mode 100644
index 0000000000000000000000000000000000000000..18fdfa579b9f1138a44d4d580220f41a01ba0dae
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll
@@ -0,0 +1,92 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu %s -o %t -filetype=obj
+; RUN: llvm-dwarfdump --debug-info %t | FileCheck %s
+
+%struct.struct_t = type { i8 }
+
+@g = dso_local global %struct.struct_t zeroinitializer, align 1, !dbg !0
+
+; CHECK-LABEL: DW_TAG_subprogram
+; CHECK: DW_AT_name ("test1")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_fbreg -1, DW_OP_deref, DW_OP_constu 0x3d, DW_OP_shl, DW_OP_constu 0x3d, DW_OP_shr, DW_OP_stack_value)
+; CHECK: DW_AT_name ("x")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_fbreg -1, DW_OP_deref, DW_OP_constu 0x39, DW_OP_shl, DW_OP_constu 0x3c, DW_OP_shra, DW_OP_stack_value)
+; CHECK: DW_AT_name ("y")
+
+define i32 @test1() !dbg !13 {
+entry:
+  %0 = alloca %struct.struct_t, align 1
+  tail call void @llvm.dbg.declare(metadata ptr %0, metadata !16, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 3)), !dbg !17
+  tail call void @llvm.dbg.declare(metadata ptr %0, metadata !18, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 3, 4)), !dbg !17
+  ret i32 0, !dbg !17
+}
+
+; CHECK-LABEL: DW_TAG_subprogram
+; CHECK: DW_AT_name ("test2")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0xff, DW_OP_and, DW_OP_constu 0x3d, DW_OP_shl, DW_OP_constu 0x3d, DW_OP_shr, DW_OP_stack_value)
+; CHECK: DW_AT_name ("x")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0xff, DW_OP_and, DW_OP_constu 0x39, DW_OP_shl, DW_OP_constu 0x3c, DW_OP_shra, DW_OP_stack_value)
+; CHECK: DW_AT_name ("y")
+
+define i8 @test2() !dbg !20 {
+entry:
+  %0 = load i8, ptr @g, align 1
+  tail call void @llvm.dbg.value(metadata i8 %0, metadata !21, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 3)), !dbg !22
+  tail call void @llvm.dbg.value(metadata i8 %0, metadata !23, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 3, 4)), !dbg !22
+  ret i8 %0, !dbg !22
+}
+
+; CHECK-LABEL: DW_TAG_subprogram
+; CHECK: DW_AT_name ("test3")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0x3f, DW_OP_shr, DW_OP_stack_value)
+; CHECK: DW_AT_name ("x")
+; CHECK: DW_TAG_variable
+; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0x3f, DW_OP_shra, DW_OP_stack_value)
+; CHECK: DW_AT_name ("y")
+
+define i64 @test3(ptr %p) !dbg !24 {
+entry:
+  %0 = load i64, ptr %p, align 8
+  tail call void @llvm.dbg.value(metadata i64 %0, metadata !25, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 63, 1)), !dbg !26
+  tail call void @llvm.dbg.value(metadata i64 %0, metadata !27, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 63, 1)), !dbg !26
+  ret i64 %0, !dbg !26
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "DW_OP_bit_piece.cpp", directory: "./")
+!4 = !{!0}
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "struct_t", file: !3, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTS8struct_t")
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !6, file: !3, baseType: !9, size: 3, flags: DIFlagBitField, extraData: i64 0)
+!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !6, file: !3, baseType: !9, size: 4, offset: 3, flags: DIFlagBitField, extraData: i64 0)
+!11 = !{i32 7, !"Dwarf Version", i32 5}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!9}
+!16 = !DILocalVariable(name: "x", scope: !13, file: !3, type: !9)
+!17 = !DILocation(line: 0, scope: !13)
+!18 = !DILocalVariable(name: "y", scope: !13, file: !3, type: !19)
+!19 = !DIBasicType(name: "signed int", size: 32, encoding: DW_ATE_signed)
+!20 = distinct !DISubprogram(name: "test2", linkageName: "test2", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2)
+!21 = !DILocalVariable(name: "x", scope: !20, file: !3, type: !9)
+!22 = !DILocation(line: 0, scope: !20)
+!23 = !DILocalVariable(name: "y", scope: !20, file: !3, type: !19)
+!24 = distinct !DISubprogram(name: "test3", linkageName: "test3", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2)
+!25 = !DILocalVariable(name: "x", scope: !24, file: !3, type: !9)
+!26 = !DILocation(line: 0, scope: !24)
+!27 = !DILocalVariable(name: "y", scope: !24, file: !3, type: !19)
diff --git a/llvm/test/DebugInfo/X86/sdag-order.ll b/llvm/test/DebugInfo/X86/sdag-order.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f959a806567913a0d88cddeb4f5dbea267e61f54
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/sdag-order.ll
@@ -0,0 +1,46 @@
+; RUN: llc %s --stop-after=finalize-isel -o - | FileCheck %s
+
+;; Check the DBG_VALUE which is salvaged from the dbg.value using an otherwised
+;; unused value is emitted at the correct position in the function.
+;; Prior (-) to patching (+), these DBG_VALUEs would sink to the bottom of the
+;; function:
+;; │   bb.1.if.then:
+;; │-    $rax = COPY %1
+;; │     DBG_VALUE 0, $noreg, !9, !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value)
+;; │+    $rax = COPY %1
+;; │     RET 0, $rax
+
+; CHECK: bb.1.if.then:
+; CHECK-NEXT: DBG_VALUE 0, $noreg, ![[#]], !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value)
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @badger(ptr sret(i64) %sret) !dbg !5 {
+entry:
+  %f.i = getelementptr i8, ptr null, i64 4
+  br label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.dbg.value(metadata ptr %f.i, metadata !9, metadata !DIExpression()), !dbg !11
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.ll", directory: "/")
+!2 = !{i32 3}
+!3 = !{i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "_ZNK1d1gEv", linkageName: "_ZNK1d1gEv", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!9}
+!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+!11 = !DILocation(line: 5, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/dwarfdump-loclist-basetyperef.test b/llvm/test/DebugInfo/dwarfdump-loclist-basetyperef.test
new file mode 100644
index 0000000000000000000000000000000000000000..1c647920a3cf21e3e0f148e9d3a85e685adeed3d
--- /dev/null
+++ b/llvm/test/DebugInfo/dwarfdump-loclist-basetyperef.test
@@ -0,0 +1,37 @@
+# REQUIRES: x86-registered-target
+
+
+# This test checks whether llvm-dwarfdump correctly handles base type
+# references when dumping the .debug_loclists section.
+
+# When dumping the .debug_loclists section, the corresponding compile unit
+# for a base type reference is not known and therefore it cannot be resolved.
+
+# prettyPrintBaseTypeRef must handle this case by printing only reduced
+# information without crashing.
+
+
+# RUN: llvm-mc %s -filetype=obj -triple=x86_64 -o %t
+# RUN: llvm-dwarfdump %t --debug-loclists | FileCheck %s
+
+# CHECK: 0x0000000c:
+# CHECK-NEXT: <default>: DW_OP_regval_type XMM0 <base_type ref: 0x2a>, DW_OP_stack_value
+
+
+	.section	.debug_loclists,"",@progbits
+	.long	.Ldebug_loc1-.Ldebug_loc0   # Length
+.Ldebug_loc0:
+	.value	0x5                         # Version
+	.byte	0x8                         # Address size
+	.byte	0                           # Segmen selector size
+	.long	0                           # Offset entry count
+
+	.byte	0x5                         # DW_LLE_default_location
+	.uleb128 0x4                        # Loc expr size
+	.byte	0xa5                        # DW_OP_regval_type
+	.uleb128 0x11                       #   XMM0
+	.uleb128 0x2a                       #   <base_type ref: 0x2a>
+	.byte	0x9f                        # DW_OP_stack_value
+
+	.byte	0                           # DW_LLE_end_of_list
+.Ldebug_loc1:
diff --git a/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test b/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test
index 5141ff6ce322cc273fce25cffeed51ef69141569..9e46570783c93c31d9f38e58e9f6fe7a0309fa31 100644
--- a/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test
+++ b/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test
@@ -1,4 +1,3 @@
-# REQUIRES: shell
 # Ensure that no realpath assumptions are made about .gnu_debuglink paths.
 
 # Copy inputs to some other location with arbitrary names, with the original
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
index dad5f8e2fc56a513a1a77cceddc262dfd39dfdca..8610645a4ca77219b5cd6ef5f67974ad58b103d4 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
@@ -1,4 +1,5 @@
-; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=1 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK
+; RUN: opt -pass-remarks-output=%t.pass-remarks -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=1 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK
+; RUN: cat %t.pass-remarks | FileCheck %s --check-prefixes=SAFETY-REMARKS
 ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=0 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=NOSAFETY,CHECK
 ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK
 ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-stack=0 -hwasan-instrument-with-calls -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=NOSTACK,CHECK
@@ -20,6 +21,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_simple
   %buf.sroa.0 = alloca i8, align 4
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
@@ -37,6 +39,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_cmpxchg
   %buf.sroa.0 = alloca i8, align 4
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
   %0 = cmpxchg ptr %buf.sroa.0, i8 1, i8 2 monotonic monotonic, align 4
@@ -54,6 +57,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_atomicrwm
   %buf.sroa.0 = alloca i8, align 4
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
   %0 = atomicrmw add ptr %buf.sroa.0, i8 1 monotonic, align 4
@@ -71,6 +75,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_use
   %buf.sroa.0 = alloca i8, align 4
   call void @use(ptr nonnull %buf.sroa.0)
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
@@ -89,6 +94,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range
   %buf.sroa.0 = alloca [10 x i8], align 4
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
@@ -106,6 +112,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range2
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
@@ -123,6 +130,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_memset
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memset
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range3
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.memset.p0.i32(ptr %ptr, i8 0, i32 1, i1 true)
@@ -138,6 +146,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_memmove
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memmove
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range4
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %ptr, i32 1, i1 true)
@@ -153,6 +162,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_memmove
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memmove
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range5
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   %buf.sroa.1 = alloca [10 x i8], align 4
@@ -171,6 +181,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
@@ -188,6 +199,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range2
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
@@ -205,6 +217,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_memset
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memset
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range3
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.memset.p0.i32(ptr %ptr, i8 0, i32 2, i1 true)
@@ -220,6 +233,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_memmove
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memmove
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range4
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %ptr, i32 2, i1 true)
@@ -235,6 +249,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_memmove
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_memmove
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range5
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   %buf.sroa.1 = alloca [10 x i8], align 4
@@ -256,6 +271,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range6
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
@@ -275,6 +291,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_potentially_out_of_range
   %buf.sroa.0 = alloca [10 x i8], align 4
   %off = call i32 @getoffset()
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 %off
@@ -293,6 +310,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_memmove
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK: call {{.*}}__hwasan_memmove
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_potentially_out_of_range2
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %a, i32 1, i1 true)
@@ -309,6 +327,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_unclear
   %buf.sroa.0 = alloca i8, align 4
   %ptr = call ptr @getptr(ptr %buf.sroa.0)
   call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %ptr)
@@ -326,6 +345,7 @@ entry:
   ; SAFETY: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_select
   %x = call ptr @getptr(ptr %a)
   %buf.sroa.0 = alloca i8, align 4
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
@@ -346,6 +366,7 @@ entry:
   ; SAFETY-NOT: call {{.*}}__hwasan_store
   ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
   ; NOSTACK-NOT: call {{.*}}__hwasan_store
+  ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_retptr
   %buf.sroa.0 = alloca i8, align 4
   call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
   %ptr = call ptr @retptr(ptr %buf.sroa.0)
diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
index 58b784779ac73221340ad0c6de032009fbe4d783..d3f82a217a17266fc98173d5fa4898ecd96259d0 100644
--- a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
@@ -1,5 +1,6 @@
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 %s | FileCheck --check-prefix=GFX1150 %s
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1152 %s | FileCheck --check-prefix=GFX1150 %s
 
 //
 // Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable
diff --git a/llvm/test/MC/ARM/dfb-neg.s b/llvm/test/MC/ARM/dfb-neg.s
index 15c44877fa6fe5af00901ea744e8b6fbbb560441..ec9192bca3ccab705ff745fd31df63b5c0b276f6 100644
--- a/llvm/test/MC/ARM/dfb-neg.s
+++ b/llvm/test/MC/ARM/dfb-neg.s
@@ -1,5 +1,7 @@
 @ RUN: not llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s
 @ RUN: not llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s
+@ RUN: not llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s
+@ RUN: not llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s
 
         dfb
 @ CHECK: error: instruction requires: full-data-barrier
diff --git a/llvm/test/MC/ARM/dfb.s b/llvm/test/MC/ARM/dfb.s
index 58477749807fee6ee165a9d402e94e1b4f03f95e..9c4768f6e245b460b59932cdecd45edab45726ce 100644
--- a/llvm/test/MC/ARM/dfb.s
+++ b/llvm/test/MC/ARM/dfb.s
@@ -1,5 +1,7 @@
 @ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-ARM
 @ RUN: llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-THUMB
+@ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-ARM
+@ RUN: llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-THUMB
 
         dfb
 @ CHECK-ARM:   dfb                             @ encoding: [0x4c,0xf0,0x7f,0xf5]
diff --git a/llvm/test/MC/ARM/invalid-armv8r.s b/llvm/test/MC/ARM/invalid-armv8r.s
new file mode 100644
index 0000000000000000000000000000000000000000..c87875a22f1444b0e4068df844333532b4ba12e3
--- /dev/null
+++ b/llvm/test/MC/ARM/invalid-armv8r.s
@@ -0,0 +1,6 @@
+@ RUN: not llvm-mc -triple=armv8r-linux-gnu -mcpu=cortex-r52 -show-encoding < %s 2>&1 | FileCheck %s -check-prefix=CHECK-NOTZ
+@ RUN: not llvm-mc -triple=armv8r-linux-gnu -mcpu=cortex-r52plus     -show-encoding < %s 2>&1 | FileCheck %s -check-prefix=CHECK-NOTZ
+
+    smc #0xf
+
+@ CHECK-NOTZ: error: instruction requires: TrustZone
diff --git a/llvm/test/MC/ARM/thumb-hints.s b/llvm/test/MC/ARM/thumb-hints.s
index b3c4cee40bbd09d3b38df581377d0633282dbe96..53b7b91237cc2cf3188322d7492b0705ce2661e7 100644
--- a/llvm/test/MC/ARM/thumb-hints.s
+++ b/llvm/test/MC/ARM/thumb-hints.s
@@ -1,3 +1,5 @@
+@ RUN: llvm-mc -triple=thumbv8r-apple-darwin -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s
+@ RUN: llvm-mc -triple=thumbv8r-apple-darwin -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s
 @ RUN: llvm-mc -triple=thumbv7-apple-darwin -show-encoding < %s | FileCheck %s
 @ RUN: llvm-mc -triple=thumbv6-apple-darwin -mcpu=cortex-m0 -show-encoding < %s | FileCheck %s 
 @ RUN: not llvm-mc -triple=thumbv6-apple-darwin -show-encoding < %s > %t 2> %t2
diff --git a/llvm/test/MC/AsmParser/layout-interdependency.s b/llvm/test/MC/AsmParser/layout-interdependency.s
deleted file mode 100644
index d275614e87e74d32b97556069364f38ce45422cf..0000000000000000000000000000000000000000
--- a/llvm/test/MC/AsmParser/layout-interdependency.s
+++ /dev/null
@@ -1,12 +0,0 @@
-# RUN: not llvm-mc --filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
-# REQUIRES: object-emission
-# UNSUPPORTED: target={{.*}}-zos{{.*}}
-
-fct_end:
-
-# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: expected assembly-time absolute expression
-.fill (data_start - fct_end), 1, 42
-# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: expected assembly-time absolute expression
-.fill (fct_end - data_start), 1, 42
-
-data_start:
diff --git a/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt b/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt
index aa8adc83c1f4dc0e84a8b439775d6e3811ec898c..ae620cb458eb4b66df11a5f8f5262781b03a1974 100644
--- a/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt
+++ b/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt
@@ -1,5 +1,7 @@
 # RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DFB
 # RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s | FileCheck %s --check-prefix=CHECK-NODFB
+# RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DFB
+# RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s | FileCheck %s --check-prefix=CHECK-NODFB
 
 # CHECK-DFB:   dfb                             @ encoding: [0xbf,0xf3,0x4c,0x8f]
 # CHECK-NODFB: dsb     #0xc                    @ encoding: [0xbf,0xf3,0x4c,0x8f]
diff --git a/llvm/test/MC/Disassembler/Sparc/sparc-v9-asi.txt b/llvm/test/MC/Disassembler/Sparc/sparc-v9-asi.txt
index 9286814552cf5f6a3a28c382e141ea1a3a30233a..1d1fdb667fafa47a2204896526219a155fdd79d6 100644
--- a/llvm/test/MC/Disassembler/Sparc/sparc-v9-asi.txt
+++ b/llvm/test/MC/Disassembler/Sparc/sparc-v9-asi.txt
@@ -32,3 +32,8 @@
 0xd5 0xf6 0x11 0x56
 # V9: casxa [%i0] #ASI_SNF_L, %l6, %o2
 0xd5 0xf6 0x11 0x76
+
+# V9: prefetcha  [%i1+3968] %asi, #one_read
+0xc3 0xee 0x6f 0x80
+# V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read
+0xc3 0xee 0x50 0x7a
diff --git a/llvm/test/MC/Disassembler/Sparc/sparc-v9.txt b/llvm/test/MC/Disassembler/Sparc/sparc-v9.txt
index da278e1832767d8142a8f5e0f68d56e7b4f7551e..d561216fec6f26beb81ef7de0e3fb358367595f2 100644
--- a/llvm/test/MC/Disassembler/Sparc/sparc-v9.txt
+++ b/llvm/test/MC/Disassembler/Sparc/sparc-v9.txt
@@ -132,11 +132,65 @@
 # CHECK: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore | #Lookaside | #MemIssue | #Sync
 0x81 0x43 0xe0 0x7f
 
-# CHECK: prefetch  [%i1+3968], 1
-0xc3,0x6e,0x6f,0x80
+# CHECK: prefetch  [%i1+3968], #n_reads
+0xc1 0x6e 0x6f 0x80
 
-# CHECK: prefetch  [%i1+%i2], 1
-0xc3,0x6e,0x40,0x1a
+# CHECK: prefetch  [%i1+3968], #one_read
+0xc3 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #n_writes
+0xc5 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #one_write
+0xc7 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #page
+0xc9 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #unified
+0xe3 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #n_reads_strong
+0xe9 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #one_read_strong
+0xeb 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #n_writes_strong
+0xed 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+3968], #one_write_strong
+0xef 0x6e 0x6f 0x80
+
+# CHECK: prefetch  [%i1+%i2], #n_reads
+0xc1 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #one_read
+0xc3 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #n_writes
+0xc5 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #one_write
+0xc7 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #page
+0xc9 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #unified
+0xe3 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #n_reads_strong
+0xe9 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #one_read_strong
+0xeb 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #n_writes_strong
+0xed 0x6e 0x40 0x1a
+
+# CHECK: prefetch  [%i1+%i2], #one_write_strong
+0xef 0x6e 0x40 0x1a
 
 # CHECK: done
 0x81,0xf0,0x00,0x00
diff --git a/llvm/test/MC/ELF/layout-interdependency.s b/llvm/test/MC/ELF/layout-interdependency.s
new file mode 100644
index 0000000000000000000000000000000000000000..5ebcdfacdbad764c939ce85ca11b9654d3b69887
--- /dev/null
+++ b/llvm/test/MC/ELF/layout-interdependency.s
@@ -0,0 +1,10 @@
+# RUN: not llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null 2>&1 | FileCheck %s
+
+fct_end:
+
+# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: invalid number of bytes
+.fill (data_start - fct_end), 1, 42
+# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: invalid number of bytes
+.fill (fct_end - data_start), 1, 42
+
+data_start:
diff --git a/llvm/test/MC/ELF/relax-recompute-align.s b/llvm/test/MC/ELF/relax-recompute-align.s
new file mode 100644
index 0000000000000000000000000000000000000000..44e1f1f9719970f521ed8f043f3abf4984a6148d
--- /dev/null
+++ b/llvm/test/MC/ELF/relax-recompute-align.s
@@ -0,0 +1,23 @@
+// RUN: llvm-mc -filetype=obj -triple i386 %s -o - | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+/// This is a case where the computed layout is not optimal. The
+// issue is that after the first jmp slides, the .align size must be
+// recomputed -- otherwise the second jump will appear to be out-of-range for a
+// 1-byte jump.
+
+// CHECK:            int3
+// CHECK-NEXT:  d2:  int3
+// CHECK:       e0:  pushal
+// CHECK:      140:  jl      0xe0
+
+L0:
+        .space 0x8a, 0x90
+	jmp	L0
+        .space (0xb3 - 0x8f), 0x90
+	jle	L2
+        .space (0xcd - 0xb5), 0x90
+	.p2align 4, 0xcc
+L1:
+        .space (0x130 - 0xd0),0x60
+	jl	L1
+L2:
diff --git a/llvm/test/MC/ELF/subsection-if.s b/llvm/test/MC/ELF/subsection-if.s
index 7f2cba6f52100ec710b532cbcc84532ccdd6bb3d..905cb5afff2ea1d21f48c4ce2ffd0bb0d2bccfe8 100644
--- a/llvm/test/MC/ELF/subsection-if.s
+++ b/llvm/test/MC/ELF/subsection-if.s
@@ -1,8 +1,10 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
 # RUN: llvm-readelf -x .text %t | FileCheck %s
-# RUN: not llvm-mc -filetype=obj -triple=x86_64 --defsym ERR=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: llvm-mc -filetype=obj -triple=x86_64 --defsym ERR=1 %s -o %t1
+# RUN: llvm-readelf -x .text %t1 | FileCheck %s --check-prefix=CHECK1
 
 # CHECK: 0x00000000 9090
+# CHECK1: 0x00000000 90909090 90
 
 .subsection 1
 661:
diff --git a/llvm/test/MC/MachO/relax-recompute-align.s b/llvm/test/MC/MachO/relax-recompute-align.s
deleted file mode 100644
index bfb17b71274e9c8c1a574fd2dc5f3ca564bc9d29..0000000000000000000000000000000000000000
--- a/llvm/test/MC/MachO/relax-recompute-align.s
+++ /dev/null
@@ -1,42 +0,0 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S - | FileCheck %s
-
-// FIXME: This is a horrible way of checking the output, we need an llvm-mc
-// based 'otool'.
-
-// This is a case where llvm-mc computes a better layout than Darwin 'as'. This
-// issue is that after the first jmp slides, the .align size must be
-// recomputed -- otherwise the second jump will appear to be out-of-range for a
-// 1-byte jump.
-
-L0:
-        .space 0x8a, 0x90
-	jmp	L0
-        .space (0xb3 - 0x8f), 0x90
-	jle	L2
-        .space (0xcd - 0xb5), 0x90
-	.align	4, 0x90
-L1:
-        .space (0x130 - 0xd0),0x90
-	jl	L1
-L2:
-
-.zerofill __DATA,__bss,_sym,4,2
-
-// CHECK: Section {
-// CHECK-NEXT:   Index: 0
-// CHECK-NEXT:   Name: __text (5F 5F 74 65 78 74 00 00 00 00 00 00 00 00 00 00)
-// CHECK-NEXT:   Segment: __TEXT (5F 5F 54 45 58 54 00 00 00 00 00 00 00 00 00 00)
-// CHECK-NEXT:   Address: 0x0
-// CHECK-NEXT:   Size: 0x132
-// CHECK-NEXT:   Offset: 340
-// CHECK-NEXT:   Alignment: 4
-// CHECK-NEXT:   RelocationOffset: 0x0
-// CHECK-NEXT:   RelocationCount: 0
-// CHECK-NEXT:   Type: Regular (0x0)
-// CHECK-NEXT:   Attributes [ (0x800004)
-// CHECK-NEXT:     PureInstructions (0x800000)
-// CHECK-NEXT:     SomeInstructions (0x4)
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Reserved1: 0x0
-// CHECK-NEXT:   Reserved2: 0x0
-// CHECK-NEXT: }
diff --git a/llvm/test/MC/RISCV/insn.s b/llvm/test/MC/RISCV/insn.s
index ddb52da182b0201fdc99ddc962c6fc63fb6e010c..4eb9a8a7ebf258b8cd128616f25a4926ac39d66c 100644
--- a/llvm/test/MC/RISCV/insn.s
+++ b/llvm/test/MC/RISCV/insn.s
@@ -38,6 +38,15 @@ target:
 # CHECK-OBJ: jalr a0, 0xa(a1)
 .insn i  JALR,  0, a0, 10(a1)
 
+# CHECK-ASM: .insn i 103, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x67,0x85,0x05,0x00]
+# CHECK-OBJ: jalr a0, 0x0(a1)
+.insn i  0x67,  0, a0, (a1)
+# CHECK-ASM: .insn i 103, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x67,0x85,0x05,0x00]
+# CHECK-OBJ: jalr a0, 0x0(a1)
+.insn i  JALR,  0, a0, (a1)
+
 # CHECK-ASM: .insn i 3, 0, a0, 4(a1)
 # CHECK-ASM: encoding: [0x03,0x85,0x45,0x00]
 # CHECK-OBJ: lb a0, 0x4(a1)
@@ -47,6 +56,15 @@ target:
 # CHECK-OBJ: lb a0, 0x4(a1)
 .insn i   LOAD,  0, a0, 4(a1)
 
+# CHECK-ASM: .insn i 3, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x03,0x85,0x05,0x00]
+# CHECK-OBJ: lb a0, 0x0(a1)
+.insn i   0x3,  0, a0, (a1)
+# CHECK-ASM: .insn i 3, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x03,0x85,0x05,0x00]
+# CHECK-OBJ: lb a0, 0x0(a1)
+.insn i   LOAD,  0, a0, (a1)
+
 # CHECK-ASM: .insn b 99, 0, a0, a1, target
 # CHECK-ASM: [0x63'A',A,0xb5'A',A]
 # CHECK-OBJ: beq a0, a1, 0x0 <target>
@@ -74,6 +92,15 @@ target:
 # CHECK-OBJ: sb a0, 0x4(a1)
 .insn s  STORE,  0, a0, 4(a1)
 
+# CHECK-ASM: .insn s 35, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x23,0x80,0xa5,0x00]
+# CHECK-OBJ: sb a0, 0x0(a1)
+.insn s  0x23,  0, a0, (a1)
+# CHECK-ASM: .insn s 35, 0, a0, 0(a1)
+# CHECK-ASM: encoding: [0x23,0x80,0xa5,0x00]
+# CHECK-OBJ: sb a0, 0x0(a1)
+.insn s  STORE,  0, a0, (a1)
+
 # CHECK-ASM: .insn u 55, a0, 4095
 # CHECK-ASM: encoding: [0x37,0xf5,0xff,0x00]
 # CHECK-OBJ: lui a0, 0xfff
diff --git a/llvm/test/MC/RISCV/insn_c.s b/llvm/test/MC/RISCV/insn_c.s
index f71d0ac60e00b89429192f0b85256dfb181f39fd..e1b3003e0200d25b58ee9eaa4c24f4884f3ffc80 100644
--- a/llvm/test/MC/RISCV/insn_c.s
+++ b/llvm/test/MC/RISCV/insn_c.s
@@ -51,11 +51,21 @@ target:
 # CHECK-OBJ: c.lw a0, 0x58(a1)
 .insn cl  0, 2, a0, 13(a1)
 
+# CHECK-ASM: .insn cl  0, 2, a0, 0
+# CHECK-ASM: encoding: [0x88,0x41]
+# CHECK-OBJ: c.lw a0, 0x0(a1)
+.insn cl  0, 2, a0, 0(a1)
+
 # CHECK-ASM: .insn cs  0, 6, a0, 13
 # CHECK-ASM: encoding: [0xa8,0xcd]
 # CHECK-OBJ: c.sw a0, 0x58(a1)
 .insn cs  0, 6, a0, 13(a1)
 
+# CHECK-ASM: .insn cs  0, 6, a0, 0
+# CHECK-ASM: encoding: [0x88,0xc1]
+# CHECK-OBJ: c.sw a0, 0x0(a1)
+.insn cs  0, 6, a0, (a1)
+
 # CHECK-ASM: .insn ca  1, 35, 0, a0, a1
 # CHECK-ASM: encoding: [0x0d,0x8d]
 # CHECK-OBJ: c.sub a0, a1
diff --git a/llvm/test/MC/RISCV/relocations.s b/llvm/test/MC/RISCV/relocations.s
index 3cad3d44a615d14d468dafaed5857081a7f80963..f5f6417487f224a6ece94618ec466b0a340fe168 100644
--- a/llvm/test/MC/RISCV/relocations.s
+++ b/llvm/test/MC/RISCV/relocations.s
@@ -185,7 +185,7 @@ auipc a0, %tlsdesc_hi(a_symbol)
 
 lw a1, %tlsdesc_load_lo(.L5)(a0)
 # RELOC: R_RISCV_TLSDESC_LOAD_LO12
-# INSTR: a1, %tlsdesc_load_lo(.L5)(a0)
+# INSTR: lw a1, %tlsdesc_load_lo(.L5)(a0)
 # FIXUP: fixup A - offset: 0, value: %tlsdesc_load_lo(.L5), kind: fixup_riscv_tlsdesc_load_lo12
 
 addi a0, a0, %tlsdesc_add_lo(.L5)
diff --git a/llvm/test/MC/RISCV/rvi-aliases-valid.s b/llvm/test/MC/RISCV/rvi-aliases-valid.s
index 098d5c132c98c1e5dfdc0d8a40191ddfec7aa495..9ac6a8a2c1e757876ab178be961a348dd80f4000 100644
--- a/llvm/test/MC/RISCV/rvi-aliases-valid.s
+++ b/llvm/test/MC/RISCV/rvi-aliases-valid.s
@@ -190,6 +190,16 @@ jalr x25, x26, 11
 # CHECK-S-OBJ-NOALIAS: jalr zero, 0(ra)
 # CHECK-S-OBJ: ret
 ret
+# CHECK-S-OBJ-NOALIAS: jalr zero, 0(s11)
+# CHECK-S-OBJ: jr s11
+jr (x27)
+# CHECK-S-OBJ-NOALIAS: jalr ra, 0(t3)
+# CHECK-S-OBJ: jalr t3
+jalr (x28)
+# CHECK-S-OBJ-NOALIAS: jalr t4, 0(t5)
+# CHECK-S-OBJ: jalr t4, t5
+jalr x29, (x30)
+
 # TODO call
 # TODO tail
 
diff --git a/llvm/test/MC/Sparc/sparc-relocations.s b/llvm/test/MC/Sparc/sparc-relocations.s
index d99ddb7e2f802d5dd4ba3b73884182ea5e0e6f10..82314e440acf6b3fa54bfd722565b745bd23cff7 100644
--- a/llvm/test/MC/Sparc/sparc-relocations.s
+++ b/llvm/test/MC/Sparc/sparc-relocations.s
@@ -10,6 +10,8 @@
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_M44 sym
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_L44 sym
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_HH22 sym
+        ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_HH22 sym
+        ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_HM10 sym
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_HM10 sym
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_LM22 sym
         ! CHECK-OBJ-NEXT: 0x{{[0-9,A-F]+}} R_SPARC_13 sym
@@ -49,10 +51,18 @@
         ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %hh(sym), kind: fixup_sparc_hh
         sethi %hh(sym), %l0
 
+        ! CHECK: sethi %hh(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
+        ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %hh(sym), kind: fixup_sparc_hh
+        sethi %uhi(sym), %l0
+
         ! CHECK: or %g1, %hm(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
         ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %hm(sym), kind: fixup_sparc_hm
         or %g1, %hm(sym), %g3
 
+        ! CHECK: or %g1, %hm(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
+        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %hm(sym), kind: fixup_sparc_hm
+        or %g1, %ulo(sym), %g3
+
         ! CHECK: sethi %lm(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
         ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %lm(sym), kind: fixup_sparc_lm
         sethi %lm(sym), %l0
diff --git a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
index a21b17552eb150ee298d06fe2741b3a716edd280..188947314f4b715416b2a4c8d6ce973d9ab99f28 100644
--- a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
+++ b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
@@ -1190,29 +1190,37 @@
         ! CHECK-NEXT:                             !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a,pn   %g1, .BB0
 
+        ! CHECK: movrz   %g1, %g2, %g3 ! encoding: [0x87,0x78,0x44,0x02]
         ! CHECK: movrz   %g1, %g2, %g3 ! encoding: [0x87,0x78,0x44,0x02]
         ! CHECK: movrlez %g1, %g2, %g3 ! encoding: [0x87,0x78,0x48,0x02]
         ! CHECK: movrlz  %g1, %g2, %g3 ! encoding: [0x87,0x78,0x4c,0x02]
         ! CHECK: movrnz  %g1, %g2, %g3 ! encoding: [0x87,0x78,0x54,0x02]
+        ! CHECK: movrnz  %g1, %g2, %g3 ! encoding: [0x87,0x78,0x54,0x02]
         ! CHECK: movrgz  %g1, %g2, %g3 ! encoding: [0x87,0x78,0x58,0x02]
         ! CHECK: movrgez %g1, %g2, %g3 ! encoding: [0x87,0x78,0x5c,0x02]
         movrz   %g1, %g2, %g3
+        movre   %g1, %g2, %g3
         movrlez %g1, %g2, %g3
         movrlz  %g1, %g2, %g3
         movrnz  %g1, %g2, %g3
+        movrne  %g1, %g2, %g3
         movrgz  %g1, %g2, %g3
         movrgez %g1, %g2, %g3
 
+        ! CHECK: movrz   %g1, 2, %g3 ! encoding: [0x87,0x78,0x64,0x02]
         ! CHECK: movrz   %g1, 2, %g3 ! encoding: [0x87,0x78,0x64,0x02]
         ! CHECK: movrlez %g1, 2, %g3 ! encoding: [0x87,0x78,0x68,0x02]
         ! CHECK: movrlz  %g1, 2, %g3 ! encoding: [0x87,0x78,0x6c,0x02]
         ! CHECK: movrnz  %g1, 2, %g3 ! encoding: [0x87,0x78,0x74,0x02]
+        ! CHECK: movrnz  %g1, 2, %g3 ! encoding: [0x87,0x78,0x74,0x02]
         ! CHECK: movrgz  %g1, 2, %g3 ! encoding: [0x87,0x78,0x78,0x02]
         ! CHECK: movrgez %g1, 2, %g3 ! encoding: [0x87,0x78,0x7c,0x02]
         movrz   %g1, 2, %g3
+        movre   %g1, 2, %g3
         movrlez %g1, 2, %g3
         movrlz  %g1, 2, %g3
         movrnz  %g1, 2, %g3
+        movrne  %g1, 2, %g3
         movrgz  %g1, 2, %g3
         movrgez %g1, 2, %g3
 
diff --git a/llvm/test/MC/Sparc/sparcv9-instructions.s b/llvm/test/MC/Sparc/sparcv9-instructions.s
index 0ca2e50989ca9610a6b8f20c7748043716c3d5cf..1b11171b7074ddaab5852f96ae1707156fcc45c9 100644
--- a/llvm/test/MC/Sparc/sparcv9-instructions.s
+++ b/llvm/test/MC/Sparc/sparcv9-instructions.s
@@ -6,6 +6,16 @@
         ! V9:      addx %g2, %g1, %g3              ! encoding: [0x86,0x40,0x80,0x01]
         addc %g2, %g1, %g3
 
+        ! V8:      error: invalid instruction mnemonic
+        ! V8-NEXT: addc %g2, 1, %g3
+        ! V9:      addx %g2, 1, %g3              ! encoding: [0x86,0x40,0xa0,0x01]
+        addc %g2, 1, %g3
+
+        ! V8:      error: invalid instruction mnemonic
+        ! V8-NEXT: addc 1, %g2, %g3
+        ! V9:      addx %g2, 1, %g3              ! encoding: [0x86,0x40,0xa0,0x01]
+        addc 1, %g2, %g3
+
         ! V8:      error: invalid instruction mnemonic
         ! V8-NEXT: addccc %g1, %g2, %g3
         ! V9:      addxcc %g1, %g2, %g3            ! encoding: [0x86,0xc0,0x40,0x02]
@@ -492,6 +502,10 @@
         wr %i0, %i1, %ccr
         ! V9: wr %i0, 1, %ccr           ! encoding: [0x85,0x86,0x20,0x01]
         wr %i0, 1, %ccr
+        ! V9: wr %i0, 1, %asr20         ! encoding: [0xa9,0x86,0x20,0x01]
+        wr %i0, 1, %set_softint
+        ! V9: wr %i0, 1, %asr21         ! encoding: [0xab,0x86,0x20,0x01]
+        wr %i0, 1, %clear_softint
 
         ! V9: st %o1, [%o0]             ! encoding: [0xd2,0x22,0x00,0x00]
         stw %o1, [%o0]
@@ -523,16 +537,146 @@
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] #ASI_SNF
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
+        ! V8:      error: invalid operand for instruction
         ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], 1
-        ! V9: prefetch  [%i1+3968], 1  ! encoding: [0xc3,0x6e,0x6f,0x80]
+        ! V9: prefetch  [%i1+3968], #one_read  ! encoding: [0xc3,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_reads
+        ! V9: prefetch  [%i1+3968], #n_reads  ! encoding: [0xc1,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #n_reads
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_read
+        ! V9: prefetch  [%i1+3968], #one_read  ! encoding: [0xc3,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #one_read
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_writes
+        ! V9: prefetch  [%i1+3968], #n_writes  ! encoding: [0xc5,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #n_writes
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_write
+        ! V9: prefetch  [%i1+3968], #one_write  ! encoding: [0xc7,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #one_write
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #page
+        ! V9: prefetch  [%i1+3968], #page  ! encoding: [0xc9,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #page
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #unified
+        ! V9: prefetch  [%i1+3968], #unified  ! encoding: [0xe3,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #unified
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_reads_strong
+        ! V9: prefetch  [%i1+3968], #n_reads_strong  ! encoding: [0xe9,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #n_reads_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_read_strong
+        ! V9: prefetch  [%i1+3968], #one_read_strong  ! encoding: [0xeb,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #one_read_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_writes_strong
+        ! V9: prefetch  [%i1+3968], #n_writes_strong  ! encoding: [0xed,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #n_writes_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_write_strong
+        ! V9: prefetch  [%i1+3968], #one_write_strong  ! encoding: [0xef,0x6e,0x6f,0x80]
+        prefetch  [ %i1 + 0xf80 ], #one_write_strong
+
+        ! V8:      error: invalid operand for instruction
         ! V8-NEXT: prefetch  [ %i1 + %i2 ], 1
-        ! V9: prefetch  [%i1+%i2], 1  ! encoding: [0xc3,0x6e,0x40,0x1a]
+        ! V9: prefetch  [%i1+%i2], #one_read  ! encoding: [0xc3,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], 1
 
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_reads
+        ! V9: prefetch  [%i1+%i2], #n_reads  ! encoding: [0xc1,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #n_reads
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_read
+        ! V9: prefetch  [%i1+%i2], #one_read  ! encoding: [0xc3,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #one_read
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_writes
+        ! V9: prefetch  [%i1+%i2], #n_writes  ! encoding: [0xc5,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #n_writes
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_write
+        ! V9: prefetch  [%i1+%i2], #one_write  ! encoding: [0xc7,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #one_write
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #page
+        ! V9: prefetch  [%i1+%i2], #page  ! encoding: [0xc9,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #page
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #unified
+        ! V9: prefetch  [%i1+%i2], #unified  ! encoding: [0xe3,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #unified
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_reads_strong
+        ! V9: prefetch  [%i1+%i2], #n_reads_strong  ! encoding: [0xe9,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #n_reads_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_read_strong
+        ! V9: prefetch  [%i1+%i2], #one_read_strong  ! encoding: [0xeb,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #one_read_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_writes_strong
+        ! V9: prefetch  [%i1+%i2], #n_writes_strong  ! encoding: [0xed,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #n_writes_strong
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_write_strong
+        ! V9: prefetch  [%i1+%i2], #one_write_strong  ! encoding: [0xef,0x6e,0x40,0x1a]
+        prefetch  [ %i1 + %i2 ], #one_write_strong
+
+        ! V8:      error: malformed ASI tag, must be a constant integer expression
+        ! V8-NEXT: prefetcha  [ %i1 + 0xf80 ] %asi, 1
+        ! V9: prefetcha [%i1+3968] %asi, #one_read    ! encoding: [0xc3,0xee,0x6f,0x80]
+        prefetcha  [ %i1 + 0xf80 ] %asi, 1
+
+        ! V8:      error: malformed ASI tag, must be a constant integer expression
+        ! V8-NEXT: prefetcha  [ %i1 + 0xf80 ] %asi, #one_read
+        ! V9: prefetcha [%i1+3968] %asi, #one_read    ! encoding: [0xc3,0xee,0x6f,0x80]
+        prefetcha  [ %i1 + 0xf80 ] %asi, #one_read
+
+        ! V8:      error: malformed ASI tag, must be a constant integer expression
+        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] #ASI_SNF, 1
+        ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
+        prefetcha  [ %i1 + %i2 ] #ASI_SNF, 1
+
+        ! V8:      error: malformed ASI tag, must be a constant integer expression
+        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] #ASI_SNF, #one_read
+        ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
+        prefetcha  [ %i1 + %i2 ] #ASI_SNF, #one_read
+
+        ! V8:      error: invalid operand for instruction
+        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] 131, 1
+        ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
+        prefetcha  [ %i1 + %i2 ] 131, 1
+
+        ! V8:      error: unexpected token
+        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] 131, #one_read
+        ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
+        prefetcha  [ %i1 + %i2 ] 131, #one_read
+
         ! V8:      error: instruction requires a CPU feature not currently enabled
         ! V8-NEXT: done
         ! V9: done      ! encoding: [0x81,0xf0,0x00,0x00]
diff --git a/llvm/test/MC/WebAssembly/reloc-pic64.s b/llvm/test/MC/WebAssembly/reloc-pic64.s
index 0f2ebba2a2f33d87713440cefe3203801dd0a5cf..4c5ec4f30666aae2f658a677d501c87495aa6b80 100644
--- a/llvm/test/MC/WebAssembly/reloc-pic64.s
+++ b/llvm/test/MC/WebAssembly/reloc-pic64.s
@@ -93,6 +93,7 @@ hidden_func:
 # CHECK-NEXT:           Index:           0
 # CHECK-NEXT:           ElemType:        FUNCREF
 # CHECK-NEXT:           Limits:
+# CHECK-NEXT:             Flags:           [ IS_64 ]
 # CHECK-NEXT:             Minimum:         0x1
 # CHECK-NEXT:       - Module:          GOT.mem
 # CHECK-NEXT:         Field:           default_data
@@ -109,7 +110,7 @@ hidden_func:
 # CHECK-NEXT:   - Type:            ELEM
 # CHECK-NEXT:     Segments:
 # CHECK-NEXT:        Offset:
-# CHECK-NEXT:          Opcode:          I32_CONST
+# CHECK-NEXT:          Opcode:          I64_CONST
 # CHECK-NEXT:          Value:           1
 # CHECK-NEXT:        Functions:       [ 5 ]
 # CHECK-NEXT:   - Type:            DATACOUNT
diff --git a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
index 7f36795aa6a9e1a9c93206aede6f46b3c3d13bf0..7512edd2749120749c997f86d3178cd82a574441 100644
--- a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
+++ b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
@@ -230,6 +230,10 @@
 # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1151 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1151 %s
 # RUN: obj2yaml %t.o.AMDGCN_GFX1151 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1151 %s
 
+# RUN: sed -e 's/<BITS>/64/' -e 's/<MACH>/AMDGCN_GFX1152/' %s | yaml2obj -o %t.o.AMDGCN_GFX1152
+# RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1152 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1152 %s
+# RUN: obj2yaml %t.o.AMDGCN_GFX1152 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1152 %s
+
 # RUN: sed -e 's/<BITS>/64/' -e 's/<MACH>/AMDGCN_GFX1200/' %s | yaml2obj -o %t.o.AMDGCN_GFX1200
 # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1200 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1200 %s
 # RUN: obj2yaml %t.o.AMDGCN_GFX1200 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1200 %s
@@ -450,6 +454,9 @@
 # ELF-AMDGCN-GFX1151:   EF_AMDGPU_MACH_AMDGCN_GFX1151 (0x4A)
 # YAML-AMDGCN-GFX1151:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1151 ]
 
+# ELF-AMDGCN-GFX1152:   EF_AMDGPU_MACH_AMDGCN_GFX1152 (0x55)
+# YAML-AMDGCN-GFX1152:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1152 ]
+
 # ELF-AMDGCN-GFX1200:   EF_AMDGPU_MACH_AMDGCN_GFX1200 (0x48)
 # YAML-AMDGCN-GFX1200:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1200 ]
 
diff --git a/llvm/test/Other/can-execute.txt b/llvm/test/Other/can-execute.txt
index 46791cb892a281d17192345536f34c2c47829cf5..37626e7357b28b70283a643681a54d32652a1b39 100644
--- a/llvm/test/Other/can-execute.txt
+++ b/llvm/test/Other/can-execute.txt
@@ -1,5 +1,4 @@
 REQUIRES: can-execute
-REQUIRES: shell
 
 This tests that we abstract two peculiarities of unix in can_execute:
 
diff --git a/llvm/test/Other/constant-fold-gep.ll b/llvm/test/Other/constant-fold-gep.ll
index d4bd24df215e85badb88a6a281bf0ab4dfa504cf..e0d535e6952730900a04789ed3fe71444aed5e09 100644
--- a/llvm/test/Other/constant-fold-gep.ll
+++ b/llvm/test/Other/constant-fold-gep.ll
@@ -104,7 +104,7 @@
 
 ; Fold GEP of a GEP. Very simple cases are folded without targetdata.
 
-; PLAIN: @Y = global ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 2)
+; PLAIN: @Y = global ptr getelementptr inbounds ([3 x { i32, i32 }], ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 1), i64 1)
 ; PLAIN: @Z = global ptr getelementptr inbounds (i32, ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 0, i64 1, i32 0), i64 1)
 ; OPT: @Y = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 48)
 ; OPT: @Z = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 12)
diff --git a/llvm/test/Other/lit-unicode.txt b/llvm/test/Other/lit-unicode.txt
index 2f40001451688af5e5ea9ab75307b76e7af62193..b375fc505b737ca27ec71b55caf6291f17fda2f1 100644
--- a/llvm/test/Other/lit-unicode.txt
+++ b/llvm/test/Other/lit-unicode.txt
@@ -1,5 +1,4 @@
 FIXME: See if we can fix this in lit by using Unicode strings.
-REQUIRES: shell
 
 RUN: echo "ようこそ" | FileCheck %s
 CHECK: {{^}}ようこそ{{$}}
diff --git a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2f0b51c410a569bef764182902d30b63afa144f5
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll
@@ -0,0 +1,443 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p constraint-elimination -S %s | FileCheck %s
+
+declare void @llvm.assume(i1)
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1235
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_start_value(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_start_value(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1235, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 1235, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1235
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known_due_to_precond_on_start_value(ptr %s, i32 %start) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known_due_to_precond_on_start_value(
+; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[PRE_C:%.*]] = icmp ule i32 [[START]], 1234
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRE_C]])
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  %pre.c = icmp ule i32 %start, 1234
+  call void @llvm.assume(i1 %pre.c)
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1235
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_precond_on_start_value(ptr %s, i32 %start) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_precond_on_start_value(
+; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[PRE_C:%.*]] = icmp ule i32 [[START]], 1236
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRE_C]])
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1236
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  %pre.c = icmp ule i32 %start, 1236
+  call void @llvm.assume(i1 %pre.c)
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1236
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_missing_precond(ptr %s, i32 %start) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_missing_precond(
+; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1236
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1236
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known(ptr %s, i1 %pre.c, i32 %x) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known(
+; CHECK-SAME: ptr [[S:%.*]], i1 [[PRE_C:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[PRE_C]], label %[[LOOP_HEADER:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[X]], %[[ENTRY]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[IV]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[U:%.*]] = icmp ult i32 [[P]], 1235
+; CHECK-NEXT:    ret i1 [[U]]
+;
+entry:
+  br i1 %pre.c, label %loop.header, label %exit
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %p = phi i32 [ %x, %entry ], [ %iv, %loop.header ], [ %iv, %loop.latch ]
+  %u = icmp ult i32 %p, 1235
+  ret i1 %u
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_successors_swapped(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_successors_swapped(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[U:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[U]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %loop.latch, label %exit
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %u = icmp ult i32 %iv, 1235
+  ret i1 %u
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[U:%.*]] = icmp ult i32 [[IV]], 1234
+; CHECK-NEXT:    ret i1 [[U]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %u = icmp ult i32 %iv, 1234
+  ret i1 %u
+}
+
+define i1 @multi_exiting_loop_eq_same_unique_exit_var_compare_known(ptr %s, i32 %N) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_var_compare_known(
+; CHECK-SAME: ptr [[S:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ule i32 [[IV]], [[N]]
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ule i32 %iv, %N
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_ne_same_unique_exit_const_compare_known(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_ne_same_unique_exit_const_compare_known(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ne i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[T]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp ne i32 %iv, 1234
+  br i1 %exitcond.not, label %loop.latch, label %exit
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %t = icmp ult i32 %iv, 1235
+  ret i1 %t
+}
+
+define i1 @multi_exiting_loop_ne_same_unique_exit_successors_swapped(ptr %s) {
+; CHECK-LABEL: define i1 @multi_exiting_loop_ne_same_unique_exit_successors_swapped(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ne i32 [[IV]], 1234
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[U:%.*]] = icmp ult i32 [[IV]], 1235
+; CHECK-NEXT:    ret i1 [[U]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %exitcond.not = icmp ne i32 %iv, 1234
+  br i1 %exitcond.not, label %exit, label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %latch.c = icmp ult i8 %0, 10
+  %iv.next = add nuw nsw i32 %iv, 1
+  br i1 %latch.c, label %loop.header, label %exit
+
+exit:
+  %u = icmp ult i32 %iv, 1235
+  ret i1 %u
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f7e21cd586e6005a8085a1fcb3da716de53abe67
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s --check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s --check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+; Split variadic functions into two functions:
+; - one equivalent to the original, same symbol etc
+; - one implementing the contents of the original but taking a valist
+; IR here is applicable to any target that uses a ptr for valist
+;
+; Defines a function with each linkage (in the order of the llvm documentation).
+; If split applies it does the same transform to each.
+; Whether split applies depends on whether the ABI is being changed or not - e.g. a weak
+; function is not normally useful to split as the contents cannot be called from elsewhere.
+; If the ABI is being rewritten then the function is still converted. Call sites tested elsewhere.
+
+; Update test checks doesn't emit checks for declares
+
+declare void @sink_valist(ptr)
+declare void @llvm.va_start(ptr)
+declare void @llvm.va_end(ptr)
+
+declare void @decl_simple(...)
+define void @defn_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_simple(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    call void @defn_simple.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for private
+define private void @defn_private_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_private_simple(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    call void @defn_private_simple.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_private_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for internal
+define internal void @defn_internal_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_internal_simple(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    call void @defn_internal_simple.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_internal_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for available_externally
+define available_externally void @available_externally_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@available_externally_simple(...) {
+; OPT-NEXT:    %va = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va)
+; OPT-NEXT:    call void @sink_valist(ptr %va)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@available_externally_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for linkonce
+define linkonce void @defn_linkonce_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_linkonce_simple(...) {
+; OPT-NEXT:    %va = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va)
+; OPT-NEXT:    call void @sink_valist(ptr %va)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_linkonce_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for weak
+define weak void @defn_weak_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_weak_simple(...) {
+; OPT-NEXT:    %va = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va)
+; OPT-NEXT:    call void @sink_valist(ptr %va)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_weak_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; common is not applicable to functions
+; appending is not applicable to functions
+
+declare extern_weak void @decl_extern_weak_simple(...)
+; no define for extern_weak
+
+; no declare for linkonce_odr
+define linkonce_odr void @defn_linkonce_odr_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_linkonce_odr_simple(...) {
+; OPT-NEXT:    %va = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va)
+; OPT-NEXT:    call void @sink_valist(ptr %va)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_linkonce_odr_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+; no declare for weak_odr
+define weak_odr void @defn_weak_odr_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_weak_odr_simple(...) {
+; OPT-NEXT:    %va = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va)
+; OPT-NEXT:    call void @sink_valist(ptr %va)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_weak_odr_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
+
+declare external void @decl_external_simple(...)
+define external void @defn_external_simple(...) {
+; OPT-LABEL: define {{[^@]+}}@defn_external_simple(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    call void @defn_external_simple.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: define {{[^@]+}}@defn_external_simple(ptr %varargs) {
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    call void @sink_valist(ptr %va)
+; ABI-NEXT:    ret void
+;
+  %va = alloca ptr, align 4
+  call void @llvm.va_start(ptr %va)
+  call void @sink_valist(ptr %va)
+  call void @llvm.va_end(ptr %va)
+  ret void
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9a86540ba2d5b706c9640fa31eae204e598a977e
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
@@ -0,0 +1,214 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s --check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s --check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+; Examples are variadic functions that return the first or the second of an int and a double
+; Split the functions into an internal equivalent that takes a va_list and a ABI preserving wrapper
+
+define i32 @variadic_int_double_get_firstz(...) {
+; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    %1 = call i32 @variadic_int_double_get_firstz.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret i32 %1
+;
+; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(ptr %varargs) {
+; ABI-NEXT:  entry:
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    %argp.cur = load ptr, ptr %va, align 4
+; ABI-NEXT:    %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+; ABI-NEXT:    store ptr %argp.next, ptr %va, align 4
+; ABI-NEXT:    %0 = load i32, ptr %argp.cur, align 4
+; ABI-NEXT:    ret i32 %0
+;
+entry:
+  %va = alloca ptr, align 4
+  call void @llvm.va_start.p0(ptr nonnull %va)
+  %argp.cur = load ptr, ptr %va, align 4
+  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+  store ptr %argp.next, ptr %va, align 4
+  %0 = load i32, ptr %argp.cur, align 4
+  call void @llvm.va_end.p0(ptr %va)
+  ret i32 %0
+}
+
+; CHECK-LABEL: define i32 @variadic_int_double_get_firstz(...) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %va_list = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr %va_list)
+; CHECK-NEXT:    %0 = tail call i32 @variadic_int_double_get_firstz.valist(ptr %va_list)
+; CHECK-NEXT:    ret i32 %0
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal i32 @variadic_int_double_get_firstz.valist(ptr noalias %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:   %va = alloca ptr, align 4
+; CHECK-NEXT:   store ptr %varargs, ptr %va, align 4
+; CHECK-NEXT:   %argp.cur = load ptr, ptr %va, align 4
+; CHECK-NEXT:   %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+; CHECK-NEXT:   store ptr %argp.next, ptr %va, align 4
+; CHECK-NEXT:   %0 = load i32, ptr %argp.cur, align 4
+; CHECK-NEXT:   ret i32 %0
+; CHECK-NEXT:  }
+
+define double @variadic_int_double_get_secondz(...) {
+; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(...) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %va_start = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
+; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
+; OPT-NEXT:    %1 = call double @variadic_int_double_get_secondz.valist(ptr %0)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    ret double %1
+;
+; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(ptr %varargs) {
+; ABI-NEXT:  entry:
+; ABI-NEXT:    %va = alloca ptr, align 4
+; ABI-NEXT:    store ptr %varargs, ptr %va, align 4
+; ABI-NEXT:    %argp.cur = load ptr, ptr %va, align 4
+; ABI-NEXT:    %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+; ABI-NEXT:    %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12
+; ABI-NEXT:    store ptr %argp.next2, ptr %va, align 4
+; ABI-NEXT:    %0 = load double, ptr %argp.next, align 4
+; ABI-NEXT:    ret double %0
+;
+entry:
+  %va = alloca ptr, align 4
+  call void @llvm.va_start.p0(ptr nonnull %va)
+  %argp.cur = load ptr, ptr %va, align 4
+  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+  %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12
+  store ptr %argp.next2, ptr %va, align 4
+  %0 = load double, ptr %argp.next, align 4
+  call void @llvm.va_end.p0(ptr %va)
+  ret double %0
+}
+
+; CHECK-LABEL: define double @variadic_int_double_get_secondz(...) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %va_list = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr %va_list)
+; CHECK-NEXT:    %0 = tail call double @variadic_int_double_get_secondz.valist(ptr %va_list)
+; CHECK-NEXT:    ret double %0
+; CHECK-NEXT:  }
+
+; CHECK-LABEL: define internal double @variadic_int_double_get_secondz.valist(ptr noalias %varargs) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %va = alloca ptr, align 4
+; CHECK-NEXT:    store ptr %varargs, ptr %va, align 4
+; CHECK-NEXT:    %argp.cur = load ptr, ptr %va, align 4
+; CHECK-NEXT:    %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+; CHECK-NEXT:    %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12
+; CHECK-NEXT:    store ptr %argp.next2, ptr %va, align 4
+; CHECK-NEXT:    %0 = load double, ptr %argp.next, align 4
+; CHECK-NEXT:    ret double %0
+; CHECK-NEXT:  }
+
+
+; CHECK-LABEL: @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store double %y, ptr %1, align 4
+; CHECK-NEXT:    %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    %cmp.i = icmp eq i32 %call, %x
+; CHECK-NEXT:    ret i1 %cmp.i
+; CHECK-NEXT:  }
+
+define zeroext i1 @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
+; OPT-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; OPT-NEXT:    store i32 %x, ptr %0, align 4
+; OPT-NEXT:    %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
+; OPT-NEXT:    store double %y, ptr %1, align 8
+; OPT-NEXT:    %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    %cmp.i = icmp eq i32 %call, %x
+; OPT-NEXT:    ret i1 %cmp.i
+;
+; ABI-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
+; ABI-NEXT:  entry:
+; ABI-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; ABI-NEXT:    store i32 %x, ptr %0, align 4
+; ABI-NEXT:    %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
+; ABI-NEXT:    store double %y, ptr %1, align 8
+; ABI-NEXT:    %call = call i32 @variadic_int_double_get_firstz(ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    %cmp.i = icmp eq i32 %call, %x
+; ABI-NEXT:    ret i1 %cmp.i
+;
+entry:
+  %call = call i32 (...) @variadic_int_double_get_firstz(i32 %x, double %y)
+  %cmp.i = icmp eq i32 %call, %x
+  ret i1 %cmp.i
+}
+
+; CHECK-LABEL: @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; CHECK-NEXT:    store i32 %x, ptr %0, align 4
+; CHECK-NEXT:    %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1
+; CHECK-NEXT:    store double %y, ptr %1, align 4
+; CHECK-NEXT:    %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    %cmp.i = fcmp oeq double %call, %y
+; CHECK-NEXT:    ret i1 %cmp.i
+; CHECK-NEXT:  }
+
+define zeroext i1 @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
+; OPT-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
+; OPT-NEXT:  entry:
+; OPT-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; OPT-NEXT:    store i32 %x, ptr %0, align 4
+; OPT-NEXT:    %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
+; OPT-NEXT:    store double %y, ptr %1, align 8
+; OPT-NEXT:    %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    %cmp.i = fcmp oeq double %call, %y
+; OPT-NEXT:    ret i1 %cmp.i
+;
+; ABI-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
+; ABI-NEXT:  entry:
+; ABI-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
+; ABI-NEXT:    store i32 %x, ptr %0, align 4
+; ABI-NEXT:    %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
+; ABI-NEXT:    store double %y, ptr %1, align 8
+; ABI-NEXT:    %call = call double @variadic_int_double_get_secondz(ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    %cmp.i = fcmp oeq double %call, %y
+; ABI-NEXT:    ret i1 %cmp.i
+;
+entry:
+  %call = call double (...) @variadic_int_double_get_secondz(i32 %x, double %y)
+  %cmp.i = fcmp oeq double %call, %y
+  ret i1 %cmp.i
+}
+
+; Declaration unchanged
+; CHECK: declare void @variadic_without_callers(...)
+declare void @variadic_without_callers(...)
+
+declare void @llvm.va_start.p0(ptr)
+declare void @llvm.va_end.p0(ptr)
diff --git a/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll
new file mode 100644
index 0000000000000000000000000000000000000000..de04c7235ad16d0c7cc52818bd612f2acb187ddc
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+declare void @vararg(...)
+@vararg_ptr = hidden global ptr @vararg, align 4
+
+%struct.libcS = type { i8, i16, i32, i32, float, double }
+
+define hidden void @fptr_single_i32(i32 noundef %x) {
+; OPT-LABEL: @fptr_single_i32(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
+; OPT-NEXT:    tail call void (...) [[TMP0]](i32 noundef [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @fptr_single_i32(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[FPTR_SINGLE_I32_VARARG:%.*]], align 16
+; ABI-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[FPTR_SINGLE_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void [[TMP0]](ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr @vararg_ptr, align 4
+  tail call void (...) %0(i32 noundef %x)
+  ret void
+}
+
+define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) {
+; OPT-LABEL: @fptr_libcS(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
+; OPT-NEXT:    tail call void (...) [[TMP0]](ptr noundef nonnull byval([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @fptr_libcS(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[FPTR_LIBCS_VARARG:%.*]], align 16
+; ABI-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
+; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false)
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[FPTR_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void [[TMP0]](ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  %0 = load volatile ptr, ptr @vararg_ptr, align 4
+  tail call void (...) %0(ptr noundef nonnull byval(%struct.libcS) align 8 %x)
+  ret void
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/intrinsics.ll b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1782c922957443ae63124a1ae834ddb0c9be1cec
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=CHECK,OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=CHECK,ABI
+; REQUIRES: webassembly-registered-target
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+
+declare void @llvm.va_copy.p0(ptr, ptr)
+
+declare void @valist(ptr noundef)
+
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+declare void @llvm.va_start.p0(ptr)
+
+declare void @llvm.va_end.p0(ptr)
+
+
+define void @start_once(...) {
+; OPT-LABEL: @start_once(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[VA_START:%.*]] = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr [[VA_START]])
+; OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4
+; OPT-NEXT:    call void @start_once.valist(ptr [[TMP0]])
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @start_once(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[S:%.*]] = alloca ptr, align 4
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S]])
+; ABI-NEXT:    store ptr [[VARARGS:%.*]], ptr [[S]], align 4
+; ABI-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S]], align 4
+; ABI-NEXT:    call void @valist(ptr noundef [[TMP0]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S]])
+; ABI-NEXT:    ret void
+;
+entry:
+  %s = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+  call void @llvm.va_start.p0(ptr nonnull %s)
+  %0 = load ptr, ptr %s, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+  ret void
+}
+
+
+define void @start_twice(...) {
+; OPT-LABEL: @start_twice(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[VA_START:%.*]] = alloca ptr, align 4
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.va_start.p0(ptr [[VA_START]])
+; OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4
+; OPT-NEXT:    call void @start_twice.valist(ptr [[TMP0]])
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @start_twice(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[S0:%.*]] = alloca ptr, align 4
+; ABI-NEXT:    [[S1:%.*]] = alloca ptr, align 4
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S0]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S1]])
+; ABI-NEXT:    store ptr [[VARARGS:%.*]], ptr [[S0]], align 4
+; ABI-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S0]], align 4
+; ABI-NEXT:    call void @valist(ptr noundef [[TMP0]])
+; ABI-NEXT:    store ptr [[VARARGS]], ptr [[S1]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S1]], align 4
+; ABI-NEXT:    call void @valist(ptr noundef [[TMP1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S0]])
+; ABI-NEXT:    ret void
+;
+entry:
+  %s0 = alloca ptr, align 4
+  %s1 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.va_start.p0(ptr nonnull %s0)
+  %0 = load ptr, ptr %s0, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.va_end.p0(ptr %s0)
+  call void @llvm.va_start.p0(ptr nonnull %s1)
+  %1 = load ptr, ptr %s1, align 4
+  call void @valist(ptr noundef %1)
+  call void @llvm.va_end.p0(ptr %s1)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+  ret void
+}
+
+define void @copy(ptr noundef %va) {
+; CHECK-LABEL: @copy(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VA_ADDR:%.*]] = alloca ptr, align 4
+; CHECK-NEXT:    [[CP:%.*]] = alloca ptr, align 4
+; CHECK-NEXT:    store ptr [[VA:%.*]], ptr [[VA_ADDR]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[CP]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[CP]], ptr [[VA_ADDR]], i32 4, i1 false)
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[CP]], align 4
+; CHECK-NEXT:    call void @valist(ptr noundef [[TMP0]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[CP]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %va.addr = alloca ptr, align 4
+  %cp = alloca ptr, align 4
+  store ptr %va, ptr %va.addr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr)
+  %0 = load ptr, ptr %cp, align 4
+  call void @valist(ptr noundef %0)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+  ret void
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/invoke.ll b/llvm/test/Transforms/ExpandVariadics/invoke.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ced2edf9274fad853d8f330b74a2528daa8b2d8e
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/invoke.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: not --crash opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s 2>&1 | FileCheck %s -check-prefixes=ERROR
+; REQUIRES: webassembly-registered-target
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+
+; ERROR: LLVM ERROR: Cannot lower callbase instruction
+
+@_ZTIi = external constant ptr
+
+; Function Attrs: mustprogress
+define hidden void @test0(i32 noundef %x) #0 personality ptr @__gxx_wasm_personality_v0 {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void (...) @may_throw(i32 noundef [[X:%.*]])
+; CHECK-NEXT:            to label [[TRY_CONT:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchswitch within none [label %catch.start] unwind to caller
+; CHECK:       catch.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = catchpad within [[TMP0]] [ptr @_ZTIi]
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call ptr @llvm.wasm.get.exception(token [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.wasm.get.ehselector(token [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.eh.typeid.for.p0(ptr nonnull @_ZTIi)
+; CHECK-NEXT:    [[MATCHES:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MATCHES]], label [[CATCH:%.*]], label [[RETHROW:%.*]]
+; CHECK:       catch:
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @__cxa_begin_catch(ptr [[TMP2]]) [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT:    call void (...) @dont_throw(i32 noundef [[X]]) [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT:    call void @__cxa_end_catch() [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT:    catchret from [[TMP1]] to label [[TRY_CONT]]
+; CHECK:       rethrow:
+; CHECK-NEXT:    call void @llvm.wasm.rethrow() [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT:    unreachable
+; CHECK:       try.cont:
+; CHECK-NEXT:    ret void
+;
+entry:
+  invoke void (...) @may_throw(i32 noundef %x)
+  to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch.start] unwind to caller
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [ptr @_ZTIi]
+  %2 = tail call ptr @llvm.wasm.get.exception(token %1)
+  %3 = tail call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = tail call i32 @llvm.eh.typeid.for.p0(ptr nonnull @_ZTIi)
+  %matches = icmp eq i32 %3, %4
+  br i1 %matches, label %catch, label %rethrow
+
+catch:                                            ; preds = %catch.start
+  %5 = call ptr @__cxa_begin_catch(ptr %2) #6 [ "funclet"(token %1) ]
+  call void (...) @dont_throw(i32 noundef %x) #6 [ "funclet"(token %1) ]
+  call void @__cxa_end_catch() #6 [ "funclet"(token %1) ]
+  catchret from %1 to label %try.cont
+
+rethrow:                                          ; preds = %catch.start
+  call void @llvm.wasm.rethrow() #5 [ "funclet"(token %1) ]
+  unreachable
+
+try.cont:                                         ; preds = %entry, %catch
+  ret void
+}
+
+declare void @may_throw(...)
+
+declare i32 @__gxx_wasm_personality_v0(...)
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn
+declare ptr @llvm.wasm.get.exception(token)
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn
+declare i32 @llvm.wasm.get.ehselector(token)
+
+; Function Attrs: nofree nosync nounwind memory(none)
+declare i32 @llvm.eh.typeid.for.p0(ptr)
+
+declare ptr @__cxa_begin_catch(ptr)
+
+; Function Attrs: nounwind
+declare void @dont_throw(...)
+
+declare void @__cxa_end_catch()
+
+; Function Attrs: noreturn
+declare void @llvm.wasm.rethrow()
+
+
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll
new file mode 100644
index 0000000000000000000000000000000000000000..85fefda36a76e57b498e620d5bd097c338de37e5
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+; CHECK: @sink
+declare void @sink(...)
+
+
+define void @pass_byval(ptr byval(i32) %b) {
+; OPT-LABEL: @pass_byval(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(ptr byval(i32) [[B:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_byval(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYVAL_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[TMP0]], ptr [[B:%.*]], i64 4, i1 false)
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(ptr byval(i32) %b)
+  ret void
+}
+
+%struct.libcS = type { i8, i16, i32, i32, float, double }
+
+define void @i32_libcS_byval(i32 %x, ptr noundef byval(%struct.libcS) align 8 %y) {
+; OPT-LABEL: @i32_libcS_byval(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[X:%.*]], ptr byval([[STRUCT_LIBCS:%.*]]) align 8 [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @i32_libcS_byval(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYVAL_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[Y:%.*]], i64 24, i1 false)
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %x, ptr byval(%struct.libcS) align 8 %y)
+  ret void
+}
+
+define void @libcS_i32_byval(ptr byval(%struct.libcS) align 8 %x, i32 %y) {
+; OPT-LABEL: @libcS_i32_byval(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(ptr byval([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]], i32 [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @libcS_i32_byval(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYVAL_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false)
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(ptr byval(%struct.libcS) align 8 %x, i32 %y)
+  ret void
+}
+
+
+define void @pass_byref(ptr byref(i32) %b) {
+; OPT-LABEL: @pass_byref(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(ptr byref(i32) [[B:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_byref(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYREF_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store ptr [[B:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(ptr byref(i32) %b)
+  ret void
+}
+
+define void @i32_libcS_byref(i32 %x, ptr noundef byref(%struct.libcS) align 8 %y) {
+; OPT-LABEL: @i32_libcS_byref(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[X:%.*]], ptr byref([[STRUCT_LIBCS:%.*]]) align 8 [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @i32_libcS_byref(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYREF_VARARG:%.*]], align 16
+; ABI-NEXT:    store ptr [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 4
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %x, ptr byref(%struct.libcS) align 8 %y)
+  ret void
+}
+
+define void @libcS_i32_byref(ptr byref(%struct.libcS) align 8 %x, i32 %y) {
+; OPT-LABEL: @libcS_i32_byref(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(ptr byref([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]], i32 [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @libcS_i32_byref(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYREF_VARARG:%.*]], align 16
+; ABI-NEXT:    store ptr [[X:%.*]], ptr [[INDIRECTALLOCA]], align 4
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(ptr byref(%struct.libcS) align 8 %x, i32 %y)
+  ret void
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8dcbb86d02d6f2fa767c8149aaaf54a2c291b6eb
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+; CHECK: @sink
+declare void @sink(...)
+
+%struct.libcS = type { i8, i16, i32, i32, float, double }
+
+define void @i32_libcS(i32 %x, %struct.libcS %y) {
+; OPT-LABEL: @i32_libcS(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[X:%.*]], [[STRUCT_LIBCS:%.*]] [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @i32_libcS(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_VARARG:%.*]], align 16
+; ABI-NEXT:    store [[STRUCT_LIBCS]] [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 8
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %x, %struct.libcS %y)
+  ret void
+}
+
+define void @libcS_i32(%struct.libcS %x, i32 %y) {
+; OPT-LABEL: @libcS_i32(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink([[STRUCT_LIBCS:%.*]] [[X:%.*]], i32 [[Y:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @libcS_i32(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_VARARG:%.*]], align 16
+; ABI-NEXT:    store [[STRUCT_LIBCS]] [[X:%.*]], ptr [[INDIRECTALLOCA]], align 8
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(%struct.libcS %x, i32 %y)
+  ret void
+}
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-integers.ll b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a1cb6811800c3b00c34896ac63ba9afefa83b400
--- /dev/null
+++ b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll
@@ -0,0 +1,345 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT
+; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI
+; REQUIRES: webassembly-registered-target
+
+; Wasm passes struct {char} as an i8 so can check the varargs passing works on integers smaller than the slot size
+
+declare void @sink(...)
+
+
+define void @pass_nothing()  {
+; OPT-LABEL: @pass_nothing(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink()
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_nothing(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_NOTHING_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink()
+  ret void
+}
+
+define void @pass_s1(i8 %x) {
+; OPT-LABEL: @pass_s1(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i8 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_s1(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S1_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i8 [[X:%.*]], ptr [[TMP0]], align 1
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i8 %x)
+  ret void
+}
+
+define void @pass_s2(i16 %x) {
+; OPT-LABEL: @pass_s2(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i16 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_s2(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S2_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i16 [[X:%.*]], ptr [[TMP0]], align 2
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i16 %x)
+  ret void
+}
+
+define void @pass_s3(i32 %x) {
+; OPT-LABEL: @pass_s3(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_s3(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S3_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %x)
+  ret void
+}
+
+define void @pass_s4(i64 %x) {
+; OPT-LABEL: @pass_s4(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i64 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_s4(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S4_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i64 [[X:%.*]], ptr [[TMP0]], align 8
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i64 %x)
+  ret void
+}
+
+define void @pass_s5(<4 x i32> %x) {
+; OPT-LABEL: @pass_s5(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(<4 x i32> [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_s5(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S5_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store <4 x i32> [[X:%.*]], ptr [[TMP0]], align 16
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(<4 x i32> %x)
+  ret void
+}
+
+define void @pass_int_s1(i32 %i, i8 %x) {
+; OPT-LABEL: @pass_int_s1(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], i8 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_int_s1(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S1_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 5, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i8 [[X:%.*]], ptr [[TMP1]], align 1
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 5, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, i8 %x)
+  ret void
+}
+
+define void @pass_int_s2(i32 %i, i16 %x) {
+; OPT-LABEL: @pass_int_s2(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], i16 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_int_s2(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S2_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 6, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i16 [[X:%.*]], ptr [[TMP1]], align 2
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 6, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, i16 %x)
+  ret void
+}
+
+define void @pass_int_s3(i32 %i, i32 %x) {
+; OPT-LABEL: @pass_int_s3(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], i32 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_int_s3(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S3_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP1]], align 4
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, i32 %x)
+  ret void
+}
+
+define void @pass_int_s4(i32 %i, i64 %x) {
+; OPT-LABEL: @pass_int_s4(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], i64 [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_int_s4(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S4_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
+; ABI-NEXT:    store i64 [[X:%.*]], ptr [[TMP1]], align 8
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, i64 %x)
+  ret void
+}
+
+define void @pass_int_s5(i32 %i, <4 x i32> %x) {
+; OPT-LABEL: @pass_int_s5(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], <4 x i32> [[X:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_int_s5(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S5_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
+; ABI-NEXT:    store <4 x i32> [[X:%.*]], ptr [[TMP1]], align 16
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, <4 x i32> %x)
+  ret void
+}
+
+define void @pass_asc(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) {
+; OPT-LABEL: @pass_asc(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i8 [[X1:%.*]], i16 [[X2:%.*]], i32 [[X3:%.*]], i64 [[X4:%.*]], <4 x i32> [[X5:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_asc(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_ASC_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 48, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i8 [[X1:%.*]], ptr [[TMP0]], align 1
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
+; ABI-NEXT:    store i16 [[X2:%.*]], ptr [[TMP1]], align 2
+; ABI-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 4
+; ABI-NEXT:    store i32 [[X3:%.*]], ptr [[TMP2]], align 4
+; ABI-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 6
+; ABI-NEXT:    store i64 [[X4:%.*]], ptr [[TMP3]], align 8
+; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 8
+; ABI-NEXT:    store <4 x i32> [[X5:%.*]], ptr [[TMP4]], align 16
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 48, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5)
+  ret void
+}
+
+define void @pass_dsc(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4) {
+; OPT-LABEL: @pass_dsc(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(<4 x i32> [[X0:%.*]], i64 [[X1:%.*]], i32 [[X2:%.*]], i16 [[X3:%.*]], i8 [[X4:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_dsc(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_DSC_VARARG:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 33, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store <4 x i32> [[X0:%.*]], ptr [[TMP0]], align 16
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i64 [[X1:%.*]], ptr [[TMP1]], align 8
+; ABI-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
+; ABI-NEXT:    store i32 [[X2:%.*]], ptr [[TMP2]], align 4
+; ABI-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3
+; ABI-NEXT:    store i16 [[X3:%.*]], ptr [[TMP3]], align 2
+; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 5
+; ABI-NEXT:    store i8 [[X4:%.*]], ptr [[TMP4]], align 1
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 33, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4)
+  ret void
+}
+
+define void @pass_multiple(i32 %i, i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) {
+; OPT-LABEL: @pass_multiple(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I:%.*]], i16 [[X2:%.*]], i64 [[X4:%.*]])
+; OPT-NEXT:    tail call void (...) @sink(i32 [[I]], i8 [[X1:%.*]], i32 [[X3:%.*]], <4 x i32> [[X5:%.*]])
+; OPT-NEXT:    ret void
+;
+; ABI-LABEL: @pass_multiple(
+; ABI-NEXT:  entry:
+; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_MULTIPLE_VARARG:%.*]], align 16
+; ABI-NEXT:    [[VARARG_BUFFER1:%.*]] = alloca [[PASS_MULTIPLE_VARARG_0:%.*]], align 16
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
+; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
+; ABI-NEXT:    store i16 [[X2:%.*]], ptr [[TMP1]], align 2
+; ABI-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3
+; ABI-NEXT:    store i64 [[X4:%.*]], ptr [[TMP2]], align 8
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER1]])
+; ABI-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 0
+; ABI-NEXT:    store i32 [[I]], ptr [[TMP3]], align 4
+; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 1
+; ABI-NEXT:    store i8 [[X1:%.*]], ptr [[TMP4]], align 1
+; ABI-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 3
+; ABI-NEXT:    store i32 [[X3:%.*]], ptr [[TMP5]], align 4
+; ABI-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 5
+; ABI-NEXT:    store <4 x i32> [[X5:%.*]], ptr [[TMP6]], align 16
+; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER1]])
+; ABI-NEXT:    ret void
+;
+entry:
+  tail call void (...) @sink(i32 %i, i16 %x2, i64 %x4)
+  tail call void (...) @sink(i32 %i, i8 %x1, i32 %x3, <4 x i32> %x5)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
index 4566865bc7c671d0992d8162f6125e6066d05bd3..7e9d12241ee362ba274a99a52cbc86f11257047e 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
@@ -1,423 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=instcombine -mtriple=amdgcn-amd-amdhsa %s | FileCheck %s
 
-; --------------------------------------------------------------------
-; llvm.amdgcn.buffer.load
-; --------------------------------------------------------------------
-
-define amdgpu_ps float @buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret float %data
-}
-
-define amdgpu_ps <1 x float> @buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v1f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <1 x float> [[DATA]]
-;
-  %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <1 x float> %data
-}
-
-define amdgpu_ps <2 x float> @buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <2 x float> %data
-}
-
-define amdgpu_ps <4 x float> @buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <4 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <4 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <2 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 3
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 2, i32 3>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <3 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps { float, float } @extract_elt0_elt1_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32_2(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x float> [[DATA]], i64 0
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    [[INS0:%.*]] = insertvalue { float, float } undef, float [[ELT0]], 0
-; CHECK-NEXT:    [[INS1:%.*]] = insertvalue { float, float } [[INS0]], float [[ELT1]], 1
-; CHECK-NEXT:    ret { float, float } [[INS1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt1 = extractelement <4 x float> %data, i32 1
-  %ins0 = insertvalue { float, float } undef, float %elt0, 0
-  %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
-  ret { float, float } %ins1
-}
-
-define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_2(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <3 x float> [[DATA]], i64 0
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 1
-; CHECK-NEXT:    [[ELT2:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    [[INS0:%.*]] = insertvalue { float, float, float } undef, float [[ELT0]], 0
-; CHECK-NEXT:    [[INS1:%.*]] = insertvalue { float, float, float } [[INS0]], float [[ELT1]], 1
-; CHECK-NEXT:    [[INS2:%.*]] = insertvalue { float, float, float } [[INS1]], float [[ELT2]], 2
-; CHECK-NEXT:    ret { float, float, float } [[INS2]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt1 = extractelement <4 x float> %data, i32 1
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
-  %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
-  %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
-  ret { float, float, float } %ins2
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 poison, i32 1>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> poison, float %elt0, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 4, i32 1>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> poison, float %elt0, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> poison, <4 x float> %data, <2 x i32> <i32 5, i32 1>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> poison, float %elt2, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> %data, <4 x float> %data, <2 x i32> <i32 0, i32 5>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <3 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps float @preserve_metadata_extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @preserve_metadata_extract_elt0_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false), !fpmath [[META0:![0-9]+]]
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
-; --------------------------------------------------------------------
-; llvm.amdgcn.buffer.load.format
-; --------------------------------------------------------------------
-
-define amdgpu_ps <1 x float> @buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_format_v1f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 true)
-; CHECK-NEXT:    ret <1 x float> [[DATA]]
-;
-  %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
-  ret <1 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_format_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 true, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-; The initial insertion point is at the extractelement
-define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[VAR1:%.*]] = bitcast <4 x float> [[TMP1]] to <2 x double>
-; CHECK-NEXT:    [[VAR2:%.*]] = extractelement <2 x double> [[VAR1]], i64 0
-; CHECK-NEXT:    ret double [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <2 x double>
-  %var2 = extractelement <2 x double> %var1, i32 0
-  ret double %var2
-}
-
-define i32 @extract0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract0_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[VAR2:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    ret i32 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <4 x i32>
-  %var2 = extractelement <4 x i32> %var1, i32 0
-  ret i32 %var2
-}
-
-define i16 @extract_lo16_0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract_lo16_0_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    [[VAR2:%.*]] = trunc i32 [[TMP1]] to i16
-; CHECK-NEXT:    ret i16 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <8 x i16>
-  %var2 = extractelement <8 x i16> %var1, i32 0
-  ret i16 %var2
-}
-
-declare float @llvm.amdgcn.buffer.load.format.f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.raw.buffer.load
 ; --------------------------------------------------------------------
@@ -665,7 +248,7 @@ define float @extract0_bitcast_raw_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32
 
 define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
 ; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
   %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -4496,248 +4079,6 @@ declare <4 x float> @llvm.amdgcn.struct.ptr.tbuffer.load.v4f32(ptr addrspace(8),
 
 declare <4 x i32> @llvm.amdgcn.struct.ptr.tbuffer.load.v4i32(ptr addrspace(8), i32, i32, i32, i32, i32) #1
 
-; --------------------------------------------------------------------
-; llvm.amdgcn.tbuffer.load
-; --------------------------------------------------------------------
-
-define amdgpu_ps float @tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret float %data
-}
-
-define amdgpu_ps <2 x float> @tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret <2 x float> %data
-}
-
-define amdgpu_ps <4 x float> @tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <4 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret <4 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <2 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 3
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 2, i32 3>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <3 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <3 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define i32 @extract0_bitcast_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract0_bitcast_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[VAR2:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    ret i32 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %var1 = bitcast <4 x float> %var to <4 x i32>
-  %var2 = extractelement <4 x i32> %var1, i32 0
-  ret i32 %var2
-}
-
-define amdgpu_ps float @preserve_metadata_extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @preserve_metadata_extract_elt0_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath [[META0]]
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-declare float @llvm.amdgcn.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.tbuffer.load.v1f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-
-declare <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample
 ; --------------------------------------------------------------------
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index 598175b08315f80dfda01fa7e597406d018dce34..af12367b9759cef7d856bcd6c3cb177191d3a2c1 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -1,423 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=instcombine -mtriple=amdgcn-amd-amdhsa %s | FileCheck %s
 
-; --------------------------------------------------------------------
-; llvm.amdgcn.buffer.load
-; --------------------------------------------------------------------
-
-define amdgpu_ps float @buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret float %data
-}
-
-define amdgpu_ps <1 x float> @buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v1f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <1 x float> [[DATA]]
-;
-  %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <1 x float> %data
-}
-
-define amdgpu_ps <2 x float> @buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <2 x float> %data
-}
-
-define amdgpu_ps <4 x float> @buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <4 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  ret <4 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <2 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 3
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 2, i32 3>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <3 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt2_elt3_buffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps { float, float } @extract_elt0_elt1_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32_2(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x float> [[DATA]], i64 0
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    [[INS0:%.*]] = insertvalue { float, float } undef, float [[ELT0]], 0
-; CHECK-NEXT:    [[INS1:%.*]] = insertvalue { float, float } [[INS0]], float [[ELT1]], 1
-; CHECK-NEXT:    ret { float, float } [[INS1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt1 = extractelement <4 x float> %data, i32 1
-  %ins0 = insertvalue { float, float } undef, float %elt0, 0
-  %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
-  ret { float, float } %ins1
-}
-
-define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_2(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <3 x float> [[DATA]], i64 0
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 1
-; CHECK-NEXT:    [[ELT2:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    [[INS0:%.*]] = insertvalue { float, float, float } undef, float [[ELT0]], 0
-; CHECK-NEXT:    [[INS1:%.*]] = insertvalue { float, float, float } [[INS0]], float [[ELT1]], 1
-; CHECK-NEXT:    [[INS2:%.*]] = insertvalue { float, float, float } [[INS1]], float [[ELT2]], 2
-; CHECK-NEXT:    ret { float, float, float } [[INS2]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt1 = extractelement <4 x float> %data, i32 1
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
-  %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
-  %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
-  ret { float, float, float } %ins2
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 poison, i32 1>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> undef, float %elt0, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 4, i32 1>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> undef, float %elt0, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> poison, <4 x float> %data, <2 x i32> <i32 5, i32 1>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
-; CHECK-NEXT:    ret <2 x float> [[RET]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt2 = extractelement <4 x float> %data, i32 2
-  %ins0 = insertelement <2 x float> undef, float %elt2, i32 0
-  %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
-  %shuf = shufflevector <4 x float> %data, <4 x float> %data, <2 x i32> <i32 0, i32 5>
-  %ret = fadd <2 x float> %ins1, %shuf
-  ret <2 x float> %ret
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt0 = extractelement <3 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt2_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps float @preserve_metadata_extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @preserve_metadata_extract_elt0_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false), !fpmath [[META0:![0-9]+]]
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
-; --------------------------------------------------------------------
-; llvm.amdgcn.buffer.load.format
-; --------------------------------------------------------------------
-
-define amdgpu_ps <1 x float> @buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @buffer_load_format_v1f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 true)
-; CHECK-NEXT:    ret <1 x float> [[DATA]]
-;
-  %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
-  ret <1 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_buffer_load_format_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 true, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-; The initial insertion point is at the extractelement
-define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[VAR1:%.*]] = bitcast <4 x float> [[TMP1]] to <2 x double>
-; CHECK-NEXT:    [[VAR2:%.*]] = extractelement <2 x double> [[VAR1]], i64 0
-; CHECK-NEXT:    ret double [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <2 x double>
-  %var2 = extractelement <2 x double> %var1, i32 0
-  ret double %var2
-}
-
-define i32 @extract0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract0_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[VAR2:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    ret i32 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <4 x i32>
-  %var2 = extractelement <4 x i32> %var1, i32 0
-  ret i32 %var2
-}
-
-define i16 @extract_lo16_0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
-; CHECK-LABEL: @extract_lo16_0_bitcast_buffer_load_format_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    [[VAR2:%.*]] = trunc i32 [[TMP1]] to i16
-; CHECK-NEXT:    ret i16 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
-  %var1 = bitcast <4 x float> %var to <8 x i16>
-  %var2 = extractelement <8 x i16> %var1, i32 0
-  ret i16 %var2
-}
-
-declare float @llvm.amdgcn.buffer.load.format.f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
-
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.raw.buffer.load
 ; --------------------------------------------------------------------
@@ -665,7 +248,7 @@ define float @extract0_bitcast_raw_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32
 
 define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
 ; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0]]
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[OFS:%.*]], i32 [[SOFS:%.*]], i32 0), !fpmath [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
   %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
@@ -4495,248 +4078,6 @@ declare <4 x float> @llvm.amdgcn.struct.ptr.tbuffer.load.v4f32(ptr addrspace(8),
 
 declare <4 x i32> @llvm.amdgcn.struct.ptr.tbuffer.load.v4i32(ptr addrspace(8), i32, i32, i32, i32, i32) #1
 
-; --------------------------------------------------------------------
-; llvm.amdgcn.tbuffer.load
-; --------------------------------------------------------------------
-
-define amdgpu_ps float @tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret float %data
-}
-
-define amdgpu_ps <2 x float> @tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret <2 x float> %data
-}
-
-define amdgpu_ps <4 x float> @tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <4 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  ret <4 x float> %data
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <2 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <4 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <4 x float> [[DATA]], i64 3
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <4 x float> %data, i32 3
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 2, i32 3>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_elt2_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <3 x float> [[DATA]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt2_elt3_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[DATA]], <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-; CHECK-NEXT:    ret <3 x float> [[SHUF]]
-;
-  %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 2, i32 3>
-  ret <3 x float> %shuf
-}
-
-define amdgpu_ps float @extract_elt0_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt0 = extractelement <3 x float> %data, i32 0
-  ret float %elt0
-}
-
-define amdgpu_ps float @extract_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x float> [[DATA]], i64 1
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 1
-  ret float %elt1
-}
-
-define amdgpu_ps float @extract_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt2_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <3 x float> [[DATA]], i64 2
-; CHECK-NEXT:    ret float [[ELT1]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %elt1 = extractelement <3 x float> %data, i32 2
-  ret float %elt1
-}
-
-define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    ret <2 x float> [[DATA]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 0, i32 1>
-  ret <2 x float> %shuf
-}
-
-define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v3f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    ret <2 x float> [[SHUF]]
-;
-  %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %shuf = shufflevector <3 x float> %data, <3 x float> poison, <2 x i32> <i32 1, i32 2>
-  ret <2 x float> %shuf
-}
-
-define i32 @extract0_bitcast_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @extract0_bitcast_tbuffer_load_v4f32(
-; CHECK-NEXT:    [[VAR:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-; CHECK-NEXT:    [[VAR2:%.*]] = bitcast float [[VAR]] to i32
-; CHECK-NEXT:    ret i32 [[VAR2]]
-;
-  %var = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
-  %var1 = bitcast <4 x float> %var to <4 x i32>
-  %var2 = extractelement <4 x i32> %var1, i32 0
-  ret i32 %var2
-}
-
-define amdgpu_ps float @preserve_metadata_extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
-; CHECK-LABEL: @preserve_metadata_extract_elt0_tbuffer_load_v2f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> [[RSRC:%.*]], i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath [[META0]]
-; CHECK-NEXT:    ret float [[DATA]]
-;
-  %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
-  %elt0 = extractelement <2 x float> %data, i32 0
-  ret float %elt0
-}
-
-declare float @llvm.amdgcn.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <1 x float> @llvm.amdgcn.tbuffer.load.v1f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-
-declare <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
-
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample
 ; --------------------------------------------------------------------
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
index 9cef4a3c7cc0f64eeb5a9741eb7bf513907b5403..65c89618e20ba0a0e669302a1d568ef733f5aae7 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
@@ -72,33 +72,6 @@ define amdgpu_ps void @image_store_mip_1d_store_insert_zeros_at_end(<8 x i32> in
   ret void
 }
 
-define amdgpu_ps void @buffer_store_format_insert_zeros_at_end(<4 x i32> inreg %a, float %vdata1, i32 %b) {
-; GCN-LABEL: @buffer_store_format_insert_zeros_at_end(
-; GCN-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[VDATA1:%.*]], i64 0
-; GCN-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
-; GCN-NEXT:    call void @llvm.amdgcn.buffer.store.format.v2f32(<2 x float> [[TMP2]], <4 x i32> [[A:%.*]], i32 [[B:%.*]], i32 0, i1 false, i1 false)
-; GCN-NEXT:    ret void
-;
-; GFX12-LABEL: @buffer_store_format_insert_zeros_at_end(
-; GFX12-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float poison, float poison, float 0.000000e+00, float 0.000000e+00>, float [[VDATA1:%.*]], i64 0
-; GFX12-NEXT:    [[NEWVDATA4:%.*]] = insertelement <4 x float> [[TMP1]], float [[VDATA1]], i64 1
-; GFX12-NEXT:    call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> [[NEWVDATA4]], <4 x i32> [[A:%.*]], i32 [[B:%.*]], i32 0, i1 false, i1 false)
-; GFX12-NEXT:    ret void
-;
-; GFXUNKNOWN-LABEL: @buffer_store_format_insert_zeros_at_end(
-; GFXUNKNOWN-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float poison, float poison, float 0.000000e+00, float 0.000000e+00>, float [[VDATA1:%.*]], i64 0
-; GFXUNKNOWN-NEXT:    [[NEWVDATA4:%.*]] = insertelement <4 x float> [[TMP1]], float [[VDATA1]], i64 1
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> [[NEWVDATA4]], <4 x i32> [[A:%.*]], i32 [[B:%.*]], i32 0, i1 false, i1 false)
-; GFXUNKNOWN-NEXT:    ret void
-;
-  %newvdata1 = insertelement <4 x float> undef, float %vdata1, i32 0
-  %newvdata2 = insertelement <4 x float> %newvdata1, float %vdata1, i32 1
-  %newvdata3 = insertelement <4 x float> %newvdata2, float 0.0, i32 2
-  %newvdata4 = insertelement <4 x float> %newvdata3, float 0.0, i32 3
-  call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %newvdata4, <4 x i32> %a, i32 %b, i32 0, i1 0, i1 0)
-  ret void
-}
-
 define amdgpu_ps void @struct_buffer_store_format_insert_zeros(<4 x i32> inreg %a, float %vdata1, i32 %b) {
 ; GCN-LABEL: @struct_buffer_store_format_insert_zeros(
 ; GCN-NEXT:    [[TMP1:%.*]] = insertelement <3 x float> <float poison, float 0.000000e+00, float poison>, float [[VDATA1:%.*]], i64 0
@@ -304,11 +277,9 @@ define amdgpu_ps void @struct_tbuffer_store_argument_insert_first(<4 x i32> inre
 }
 
 declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) #2
-declare void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1
 declare void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #2
 declare void @llvm.amdgcn.struct.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #0
-declare void @llvm.amdgcn.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
 declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #0
 declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
 declare void @llvm.amdgcn.image.store.3d.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll
index 32bd7a37053ed6a97f9da9baef66facf7b2839c9..0cf7cd97d8ff4d16238f04963b559a56c32dc8fc 100644
--- a/llvm/test/Transforms/InstCombine/abs-1.ll
+++ b/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -852,11 +852,8 @@ define i8 @abs_diff_signed_sgt_nuw_extra_use3(i8 %a, i8 %b) {
 
 define i32 @abs_diff_signed_slt_swap_wrong_pred1(i32 %a, i32 %b) {
 ; CHECK-LABEL: @abs_diff_signed_slt_swap_wrong_pred1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SUB_BA:%.*]] = sub nsw i32 [[B]], [[A]]
-; CHECK-NEXT:    [[SUB_AB:%.*]] = sub nsw i32 [[A]], [[B]]
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB_BA]], i32 [[SUB_AB]]
-; CHECK-NEXT:    ret i32 [[COND]]
+; CHECK-NEXT:    [[SUB_AB:%.*]] = sub nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB_AB]]
 ;
   %cmp = icmp eq i32 %a, %b
   %sub_ba = sub nsw i32 %b, %a
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index 25087fef68a11793a1e8b8c8db88ef445817d255..239e14682c3069730181fb173a957232de005d73 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -1510,6 +1510,46 @@ define i8 @add_like_or_t2_extrause(i8 %x) {
   %r = add i8 %i1, 42
   ret i8 %r
 }
+define i8 @fold_add_constant_preserve_nsw(i8 %x) {
+; CHECK-LABEL: @fold_add_constant_preserve_nsw(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X:%.*]], -120
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %or = or disjoint i8 %x, -128
+  %add = add nsw i8 %or, 8
+  ret i8 %add
+}
+define i8 @fold_add_constant_no_nsw(i8 %x) {
+; CHECK-LABEL: @fold_add_constant_no_nsw(
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X:%.*]], 120
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %or = or disjoint i8 %x, -128
+  %add = add nsw i8 %or, -8
+  ret i8 %add
+}
+define i8 @fold_add_constant_preserve_nuw(i8 %x) {
+; CHECK-LABEL: @fold_add_constant_preserve_nuw(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[X:%.*]], -116
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %or = or disjoint i8 %x, 128
+  %add = add nuw i8 %or, 12
+  ret i8 %add
+}
+define i32 @sdiv_to_udiv(i32 %arg0, i32 %arg1) {
+; CHECK-LABEL: @sdiv_to_udiv(
+; CHECK-NEXT:    [[T0:%.*]] = shl nuw nsw i32 [[ARG0:%.*]], 8
+; CHECK-NEXT:    [[T2:%.*]] = add nuw nsw i32 [[T0]], 6242049
+; CHECK-NEXT:    [[T3:%.*]] = udiv i32 [[T2]], 192
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = shl nuw nsw i32 %arg0, 8
+  %t1 = or disjoint i32 %t0, 1
+  %t2 = add nuw nsw i32 %t1, 6242048
+  %t3 = sdiv i32 %t2, 192
+  ret i32 %t3
+}
 
 define i8 @add_like_or_disjoint(i8 %x) {
 ; CHECK-LABEL: @add_like_or_disjoint(
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
index ae503bfb1cfe285d3fb0e79aa9916b7af8483318..e103fe94409869cff31ae3706a3307a3bc8bdbb2 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
@@ -98,8 +98,7 @@ declare i8 @gen8()
 define i1 @c0() {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
-; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[X]], -1
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %x = call i8 @gen8()
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
index d1dd411ee86b312bce765ba14dd6b053bec530f3..bbd733e86a32deba53234bbf73f5dc33eb03b2b3 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
@@ -125,8 +125,7 @@ define i1 @oneuse0() {
 
 define i1 @c0(i8 %x) {
 ; CHECK-LABEL: @c0(
-; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
-; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %tmp0 = and i8 %x, 3
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
index 4bed21a525f05287ea661fe55e799722e68ab98b..b167c8ad25aa94a9020709e45bd388501d3a4aa4 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
@@ -113,8 +113,7 @@ define i1 @oneuse0() {
 
 define i1 @c0(i8 %x) {
 ; CHECK-LABEL: @c0(
-; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
-; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[X:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %tmp0 = and i8 %x, 3
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
index 8415204a4915a29c75e924d3cd1c1363824f1c0e..828150244773282c19f81060ee126b441e5e5912 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
@@ -108,8 +108,7 @@ declare i8 @gen8()
 define i1 @c0() {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
-; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %x = call i8 @gen8()
diff --git a/llvm/test/Transforms/InstCombine/gepgep.ll b/llvm/test/Transforms/InstCombine/gepgep.ll
index d2a0e1db392ba0f7cef6e61b5b8faf2a6006fd7e..6b6a1597d560bea08373a65490190d9c54a01ab6 100644
--- a/llvm/test/Transforms/InstCombine/gepgep.ll
+++ b/llvm/test/Transforms/InstCombine/gepgep.ll
@@ -10,7 +10,7 @@ declare void @use(ptr)
 
 define void @f() {
 ; CHECK-LABEL: define void @f() {
-; CHECK-NEXT:    call void @use(ptr getelementptr (i8, ptr @buffer, i64 add (i64 sub (i64 0, i64 ptrtoint (ptr @buffer to i64)), i64 127)))
+; CHECK-NEXT:    call void @use(ptr getelementptr (i8, ptr getelementptr (i8, ptr @buffer, i64 add (i64 sub (i64 0, i64 ptrtoint (ptr @buffer to i64)), i64 63)), i64 64))
 ; CHECK-NEXT:    ret void
 ;
   call void @use(ptr getelementptr (i8, ptr getelementptr (i8, ptr @buffer, i64 add (i64 sub (i64 0, i64 ptrtoint (ptr @buffer to i64)), i64 63)), i64 64))
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index d5cfb74a44430131341a373aab356a078a37c5bf..871913610429b30a2ff97fb7e18e9c49343665cc 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -1716,10 +1716,9 @@ if.else:
 
 @g = external global i8
 
-; FIXME: This is a miscompile
 define ptr @constexpr_gep_of_gep_with_narrow_type() {
 ; CHECK-LABEL: @constexpr_gep_of_gep_with_narrow_type(
-; CHECK-NEXT:    ret ptr getelementptr (i8, ptr @g, i64 -2)
+; CHECK-NEXT:    ret ptr getelementptr (i8, ptr @g, i64 254)
 ;
   ret ptr getelementptr (i8, ptr getelementptr (i8, ptr @g, i8 127), i8 127)
 }
diff --git a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll
index 8bb7fd0e522cb08e30a04bb16f85643694ef5090..0aace5f52c96c969e787be42240e16945035a5df 100644
--- a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll
@@ -7,8 +7,8 @@ define i1 @src_is_mask_zext(i16 %x_in, i8 %y) {
 ; CHECK-LABEL: @src_is_mask_zext(
 ; CHECK-NEXT:    [[M_IN:%.*]] = lshr i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = zext i8 [[M_IN]] to i16
-; CHECK-NEXT:    [[X:%.*]] = xor i16 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i16 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i16 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i16 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i16 %x_in, 123
@@ -83,8 +83,8 @@ define i1 @src_is_mask_and(i8 %x_in, i8 %y, i8 %z) {
 ; CHECK-NEXT:    [[MY:%.*]] = lshr i8 7, [[Y:%.*]]
 ; CHECK-NEXT:    [[MZ:%.*]] = lshr i8 -1, [[Z:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = and i8 [[MY]], [[MZ]]
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -121,8 +121,8 @@ define i1 @src_is_mask_or(i8 %x_in, i8 %y) {
 ; CHECK-LABEL: @src_is_mask_or(
 ; CHECK-NEXT:    [[MY:%.*]] = lshr i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = and i8 [[MY]], 7
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -138,8 +138,8 @@ define i1 @src_is_mask_xor(i8 %x_in, i8 %y) {
 ; CHECK-LABEL: @src_is_mask_xor(
 ; CHECK-NEXT:    [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[MASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -173,8 +173,8 @@ define i1 @src_is_mask_select(i8 %x_in, i8 %y, i1 %cond) {
 ; CHECK-NEXT:    [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[MASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -249,8 +249,8 @@ define i1 @src_is_mask_lshr(i8 %x_in, i8 %y, i8 %z, i1 %cond) {
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[SMASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15
 ; CHECK-NEXT:    [[MASK:%.*]] = lshr i8 [[SMASK]], [[Z:%.*]]
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -269,8 +269,8 @@ define i1 @src_is_mask_ashr(i8 %x_in, i8 %y, i8 %z, i1 %cond) {
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[SMASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15
 ; CHECK-NEXT:    [[MASK:%.*]] = ashr i8 [[SMASK]], [[Z:%.*]]
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -287,8 +287,8 @@ define i1 @src_is_mask_p2_m1(i8 %x_in, i8 %y) {
 ; CHECK-LABEL: @src_is_mask_p2_m1(
 ; CHECK-NEXT:    [[P2ORZ:%.*]] = shl i8 2, [[Y:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = add i8 [[P2ORZ]], -1
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -304,8 +304,8 @@ define i1 @src_is_mask_umax(i8 %x_in, i8 %y) {
 ; CHECK-NEXT:    [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[MASK:%.*]] = call i8 @llvm.umax.i8(i8 [[YMASK]], i8 3)
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -324,8 +324,8 @@ define i1 @src_is_mask_umin(i8 %x_in, i8 %y, i8 %z) {
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[ZMASK:%.*]] = lshr i8 15, [[Z:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = call i8 @llvm.umin.i8(i8 [[YMASK]], i8 [[ZMASK]])
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -364,8 +364,8 @@ define i1 @src_is_mask_smax(i8 %x_in, i8 %y) {
 ; CHECK-NEXT:    [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[MASK:%.*]] = call i8 @llvm.smax.i8(i8 [[YMASK]], i8 -1)
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -383,8 +383,8 @@ define i1 @src_is_mask_smin(i8 %x_in, i8 %y) {
 ; CHECK-NEXT:    [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]]
 ; CHECK-NEXT:    [[MASK:%.*]] = call i8 @llvm.smin.i8(i8 [[YMASK]], i8 0)
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -401,8 +401,8 @@ define i1 @src_is_mask_bitreverse_not_mask(i8 %x_in, i8 %y) {
 ; CHECK-LABEL: @src_is_mask_bitreverse_not_mask(
 ; CHECK-NEXT:    [[NMASK:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    [[MASK:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[NMASK]])
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[MASK]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -455,9 +455,9 @@ define i1 @src_is_notmask_shl(i8 %x_in, i8 %y, i1 %cond) {
 define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) {
 ; CHECK-LABEL: @src_is_notmask_x_xor_neg_x(
 ; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[NEG_Y:%.*]] = add i8 [[Y:%.*]], -1
-; CHECK-NEXT:    [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]]
-; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 7
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i8 [[TMP1]], [[Y]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[TMP2]], i8 7
 ; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[TMP3]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -473,9 +473,9 @@ define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) {
 define i1 @src_is_notmask_x_xor_neg_x_inv(i8 %x_in, i8 %y, i1 %cond) {
 ; CHECK-LABEL: @src_is_notmask_x_xor_neg_x_inv(
 ; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[NEG_Y:%.*]] = add i8 [[Y:%.*]], -1
-; CHECK-NEXT:    [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]]
-; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 7
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i8 [[TMP1]], [[Y]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[TMP2]], i8 7
 ; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[X]], [[TMP3]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -625,9 +625,7 @@ define i1 @src_is_notmask_xor_fail(i8 %x_in, i8 %y) {
 
 define i1 @src_is_mask_const_slt(i8 %x_in) {
 ; CHECK-LABEL: @src_is_mask_const_slt(
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 7
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[X]], [[AND]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[X_IN:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
@@ -650,9 +648,7 @@ define i1 @src_is_mask_const_sgt(i8 %x_in) {
 
 define i1 @src_is_mask_const_sle(i8 %x_in) {
 ; CHECK-LABEL: @src_is_mask_const_sle(
-; CHECK-NEXT:    [[X:%.*]] = xor i8 [[X_IN:%.*]], 123
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 31
-; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[AND]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[X_IN:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %x = xor i8 %x_in, 123
diff --git a/llvm/test/Transforms/InstCombine/icmp-equality-xor.ll b/llvm/test/Transforms/InstCombine/icmp-equality-xor.ll
index f5d5ef32c81e81cc135eb3aceee9eb4f309f5167..e8a78df6d5f7560ac4b86399f92a800cabddffa1 100644
--- a/llvm/test/Transforms/InstCombine/icmp-equality-xor.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-equality-xor.ll
@@ -88,7 +88,7 @@ define i1 @cmpeq_xor_cst1_commuted(i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 10
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-  %b2 = mul i32 %b, %b ; thwart complexity-based canonicalization
+  %b2 = mul i32 %b, %b  ; thwart complexity-based canonicalization
   %c = xor i32 %a, 10
   %cmp = icmp eq i32 %b2, %c
   ret i1 %cmp
@@ -145,3 +145,43 @@ entry:
   %cmp = icmp ne <2 x i8> %xor, <i8 9, i8 79>
   ret <2 x i1> %cmp
 }
+
+declare void @use.i8(i8)
+define i1 @fold_xorC_eq0_multiuse(i8 %x, i8 %y) {
+; CHECK-LABEL: @fold_xorC_eq0_multiuse(
+; CHECK-NEXT:    [[XX:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use.i8(i8 [[XX]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xx = xor i8 %x, %y
+  %r = icmp eq i8 %xx, 0
+  call void @use.i8(i8 %xx)
+  ret i1 %r
+}
+
+define i1 @fold_xorC_eq1_multiuse_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @fold_xorC_eq1_multiuse_fail(
+; CHECK-NEXT:    [[XX:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[XX]], 1
+; CHECK-NEXT:    call void @use.i8(i8 [[XX]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xx = xor i8 %x, %y
+  %r = icmp eq i8 %xx, 1
+  call void @use.i8(i8 %xx)
+  ret i1 %r
+}
+
+define i1 @fold_xorC_neC_multiuse(i8 %x) {
+; CHECK-LABEL: @fold_xorC_neC_multiuse(
+; CHECK-NEXT:    [[XX:%.*]] = xor i8 [[X:%.*]], 45
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X]], 110
+; CHECK-NEXT:    call void @use.i8(i8 [[XX]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xx = xor i8 %x, 45
+  %r = icmp ne i8 %xx, 67
+  call void @use.i8(i8 %xx)
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll
index 0f26be12c39cc6813c357e365dd293fc76284f5f..75badabda01aeba85384da215377ad2bce49a411 100644
--- a/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll
@@ -58,7 +58,7 @@ define i1 @icmp_sge_x_negy(i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[CY:%.*]] = icmp slt i8 [[Y:%.*]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CY]])
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sge i8 [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq i8 [[AND]], [[X]]
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
   %cy = icmp slt i8 %y, 0
@@ -74,7 +74,7 @@ define i1 @icmp_slt_x_negy(i8 %x, i8 %y) {
 ; CHECK-NEXT:    br i1 [[CY]], label [[NEGY:%.*]], label [[POSY:%.*]]
 ; CHECK:       negy:
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp slt i8 [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ne i8 [[AND]], [[X]]
 ; CHECK-NEXT:    ret i1 [[Z]]
 ; CHECK:       posy:
 ; CHECK-NEXT:    [[R:%.*]] = call i1 @barrier()
@@ -116,10 +116,7 @@ posy:
 
 define i1 @icmp_sle_x_negy(i8 %x, i8 %yy) {
 ; CHECK-LABEL: @icmp_sle_x_negy(
-; CHECK-NEXT:    [[Y:%.*]] = or i8 [[YY:%.*]], -128
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[Y]], [[X:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sle i8 [[AND]], [[X]]
-; CHECK-NEXT:    ret i1 [[Z]]
+; CHECK-NEXT:    ret i1 true
 ;
   %y = or i8 %yy, 128
   %and = and i8 %y, %x
@@ -129,10 +126,7 @@ define i1 @icmp_sle_x_negy(i8 %x, i8 %yy) {
 
 define <2 x i1> @icmp_sgt_x_negy(<2 x i8> %x, <2 x i8> %yy) {
 ; CHECK-LABEL: @icmp_sgt_x_negy(
-; CHECK-NEXT:    [[Y:%.*]] = or <2 x i8> [[YY:%.*]], <i8 -128, i8 -128>
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> [[Y]], [[X:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sgt <2 x i8> [[AND]], [[X]]
-; CHECK-NEXT:    ret <2 x i1> [[Z]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %y = or <2 x i8> %yy, <i8 128, i8 128>
   %and = and <2 x i8> %y, %x
@@ -155,9 +149,7 @@ define <2 x i1> @icmp_sgt_x_negy_fail_partial(<2 x i8> %x, <2 x i8> %yy) {
 
 define <2 x i1> @icmp_sle_x_posy(<2 x i8> %x, <2 x i8> %yy) {
 ; CHECK-LABEL: @icmp_sle_x_posy(
-; CHECK-NEXT:    [[Y:%.*]] = and <2 x i8> [[YY:%.*]], <i8 127, i8 127>
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> [[Y]], [[X:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sle <2 x i8> [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 -1, i8 -1>
 ; CHECK-NEXT:    ret <2 x i1> [[Z]]
 ;
   %y = and <2 x i8> %yy, <i8 127, i8 127>
@@ -183,8 +175,7 @@ define i1 @icmp_sgt_x_posy(i8 %x, i8 %y) {
 ; CHECK-LABEL: @icmp_sgt_x_posy(
 ; CHECK-NEXT:    [[CY:%.*]] = icmp sgt i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CY]])
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i8 [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i8 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
   %cy = icmp sge i8 %y, 0
@@ -196,9 +187,7 @@ define i1 @icmp_sgt_x_posy(i8 %x, i8 %y) {
 
 define <2 x i1> @icmp_sgt_negx_y(<2 x i8> %xx, <2 x i8> %y) {
 ; CHECK-LABEL: @icmp_sgt_negx_y(
-; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 -128, i8 -128>
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sgt <2 x i8> [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
 ; CHECK-NEXT:    ret <2 x i1> [[Z]]
 ;
   %x = or <2 x i8> %xx, <i8 128, i8 128>
@@ -211,8 +200,7 @@ define i1 @icmp_sle_negx_y(i8 %x, i8 %y) {
 ; CHECK-LABEL: @icmp_sle_negx_y(
 ; CHECK-NEXT:    [[CX:%.*]] = icmp slt i8 [[X:%.*]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CX]])
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = icmp sle i8 [[AND]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i8 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
   %cx = icmp slt i8 %x, 0
@@ -239,9 +227,9 @@ define i1 @icmp_sle_negx_y_fail_maybe_zero(i8 %x, i8 %y) {
 
 define i1 @icmp_eq_x_invertable_y_todo(i8 %x, i1 %y) {
 ; CHECK-LABEL: @icmp_eq_x_invertable_y_todo(
-; CHECK-NEXT:    [[YY:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 -25
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[YY]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 -25
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %yy = select i1 %y, i8 7, i8 24
@@ -252,8 +240,8 @@ define i1 @icmp_eq_x_invertable_y_todo(i8 %x, i1 %y) {
 
 define i1 @icmp_eq_x_invertable_y(i8 %x, i8 %y) {
 ; CHECK-LABEL: @icmp_eq_x_invertable_y(
-; CHECK-NEXT:    [[AND:%.*]] = and i8 [[YY:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %yy = xor i8 %y, -1
diff --git a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll
index 26f53cb4807efbfaa7fae6457928995d13c3144e..304874645d5dc9fff8093937be428cdffc05963b 100644
--- a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll
@@ -373,3 +373,29 @@ define i1 @pr64610(ptr %b) {
   %r = icmp ugt i32 %or, %s
   ret i1 %r
 }
+
+define i1 @icmp_eq_x_invertable_y2_todo(i8 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @icmp_eq_x_invertable_y2_todo(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 [[Z:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %zz = xor i8 %z, -1
+  %yy = select i1 %y, i8 7, i8 %zz
+  %or = or i8 %x, %yy
+  %r = icmp eq i8 %yy, %or
+  ret i1 %r
+}
+
+define i1 @icmp_eq_x_invertable_y2(i8 %x, i8 %y) {
+; CHECK-LABEL: @icmp_eq_x_invertable_y2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %yy = xor i8 %y, -1
+  %or = or i8 %x, %yy
+  %r = icmp eq i8 %yy, %or
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-or.ll b/llvm/test/Transforms/InstCombine/icmp-or.ll
index 1f9db5e5db9aadc1987ca0d4e86e2dcefdcdef03..bedaf591fb070ef14a320d95e800c619289124ae 100644
--- a/llvm/test/Transforms/InstCombine/icmp-or.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-or.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+; RUN: opt < %s -passes='instcombine<no-verify-fixpoint>' -S | FileCheck %s
 
 declare void @use(i8)
 
@@ -428,13 +428,17 @@ define i1 @icmp_or_xor_2_ne_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
 
 ; negative test - xor multiuse
 
+; NB: This requires more than 1 iteration to simplify. After we
+; simplify `%cmp_1 = icmp eq i64 %xor, 0`, `%xor = xor i64 %x1, %y1`
+; has one use which allows for complete simplification (rooted on
+; `%or1 = or i1 %cmp, %cmp_1` so we don't end up adding it back).
 define i1 @icmp_or_xor_2_3_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
 ; CHECK-LABEL: @icmp_or_xor_2_3_fail(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i64 [[X1:%.*]], [[Y1:%.*]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i64 [[X2:%.*]], [[Y2:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i64 [[XOR]], [[XOR1]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OR]], 0
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[XOR]], 0
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[X1]], [[Y1]]
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[CMP]], [[CMP_1]]
 ; CHECK-NEXT:    ret i1 [[OR1]]
 ;
@@ -455,7 +459,7 @@ define i1 @icmp_or_xor_2_4_fail(i64 %x1, i64 %y1, i64 %x2, i64 %y2) {
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i64 [[X2:%.*]], [[Y2:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i64 [[XOR]], [[XOR1]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OR]], 0
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[XOR1]], 0
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[X2]], [[Y2]]
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[CMP]], [[CMP_1]]
 ; CHECK-NEXT:    ret i1 [[OR1]]
 ;
@@ -955,7 +959,7 @@ define i1 @icmp_or_xor_with_sub_3_6(i64 %x1, i64 %y1, i64 %x2, i64 %y2, i64 %x3,
 
 define i1 @or_disjoint_with_constants(i8 %x) {
 ; CHECK-LABEL: @or_disjoint_with_constants(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP1:%.*]], 18
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 18
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %or = or disjoint i8 %x, 1
@@ -966,8 +970,8 @@ define i1 @or_disjoint_with_constants(i8 %x) {
 
 define i1 @or_disjoint_with_constants2(i8 %x) {
 ; CHECK-LABEL: @or_disjoint_with_constants2(
-; CHECK-NEXT:    [[OR:%.*]] = or disjoint i8 [[TMP1:%.*]], 5
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP1]], 66
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X]], 66
 ; CHECK-NEXT:    call void @use(i8 [[OR]])
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 4dbe2fc88ff71c8247f6a5059f85865a2b4821c5..8fc4a40141931d633c7900fcffaa99359fcbe0ce 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -5255,6 +5255,16 @@ define i1 @test_icmp_shl_nuw(i64 %x) {
   ret i1 %cmp
 }
 
+define i1 @test_icmp_shl_nuw_i31(i31 %x) {
+; CHECK-LABEL: @test_icmp_shl_nuw_i31(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i31 [[X:%.*]], 250
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nuw i31 %x, 23
+  %cmp = icmp ugt i31 %shl, -50331648
+  ret i1 %cmp
+}
+
 define i1 @test_icmp_shl_nsw(i64 %x) {
 ; CHECK-LABEL: @test_icmp_shl_nsw(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[X:%.*]], 3
@@ -5265,6 +5275,17 @@ define i1 @test_icmp_shl_nsw(i64 %x) {
   ret i1 %cmp
 }
 
+define i1 @test_icmp_shl_nsw_i31(i31 %x) {
+; CHECK-LABEL: @test_icmp_shl_nsw_i31(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i31 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[TMP1]], -6
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i31 %x, 23
+  %cmp = icmp ugt i31 %shl, -50331648
+  ret i1 %cmp
+}
+
 define <2 x i1> @test_icmp_shl_vec(<2 x i64> %x) {
 ; CHECK-LABEL: @test_icmp_shl_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
@@ -5295,3 +5316,13 @@ define i1 @test_icmp_shl_sgt(i64 %x) {
   %cmp = icmp sgt i64 %shl, 8589934591
   ret i1 %cmp
 }
+
+define i1 @pr94897(i32 range(i32 -2147483648, 0) %x) {
+; CHECK-LABEL: @pr94897(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i32 %x, 24
+  %cmp = icmp ugt i32 %shl, -50331648
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/pow-1.ll b/llvm/test/Transforms/InstCombine/pow-1.ll
index c52b97c38893b3d827331946c98da533cda43339..44802f9caa6ea5e9fa79dc52f526f0369b9d5ba0 100644
--- a/llvm/test/Transforms/InstCombine/pow-1.ll
+++ b/llvm/test/Transforms/InstCombine/pow-1.ll
@@ -19,10 +19,10 @@
 ; NOTE: The readonly attribute on the pow call should be preserved
 ; in the cases below where pow is transformed into another function call.
 
-declare float @powf(float, float) nounwind readonly
+declare float @powf(float, float)
 declare float @llvm.pow.f32(float, float)
 declare float @llvm.fabs.f32(float)
-declare double @pow(double, double) nounwind readonly
+declare double @pow(double, double)
 declare double @llvm.pow.f64(double, double)
 declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>) nounwind readonly
 declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) nounwind readonly
@@ -62,6 +62,38 @@ define float @test_simplify1(float %x) {
   ret float %retval
 }
 
+define float @test_simplify1_noerrno(float %x) {
+; ANY-LABEL: define float @test_simplify1_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    ret float 1.000000e+00
+;
+; VC32-LABEL: define float @test_simplify1_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+00, float [[X]]) #[[ATTR2:[0-9]+]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @test_simplify1_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+00, float [[X]]) #[[ATTR2:[0-9]+]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @test_simplify1_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    ret float 1.000000e+00
+;
+; VC83-LABEL: define float @test_simplify1_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    ret float 1.000000e+00
+;
+; NOLIB-LABEL: define float @test_simplify1_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+00, float [[X]]) #[[ATTR2:[0-9]+]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 1.0, float %x) #0
+  ret float %retval
+}
+
 define <2 x float> @test_simplify1v(<2 x float> %x) {
 ; CHECK-LABEL: define <2 x float> @test_simplify1v(
 ; CHECK-SAME: <2 x float> [[X:%.*]]) {
@@ -85,6 +117,20 @@ define double @test_simplify2(double %x) {
   ret double %retval
 }
 
+define double @test_simplify2_noerrno(double %x) {
+; LIB-LABEL: define double @test_simplify2_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    ret double 1.000000e+00
+;
+; NOLIB-LABEL: define double @test_simplify2_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+00, double [[X]]) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double 1.0, double %x) #0
+  ret double %retval
+}
+
 define <2 x double> @test_simplify2v(<2 x double> %x) {
 ; CHECK-LABEL: define <2 x double> @test_simplify2v(
 ; CHECK-SAME: <2 x double> [[X:%.*]]) {
@@ -131,35 +177,70 @@ define float @test_simplify3(float %x) {
   ret float %retval
 }
 
-define double @test_simplify3n(double %x) {
-; ANY-LABEL: define double @test_simplify3n(
+define float @test_simplify3_noerrno(float %x) {
+; ANY-LABEL: define float @test_simplify3_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[X]])
+; ANY-NEXT:    ret float [[EXP2]]
+;
+; VC32-LABEL: define float @test_simplify3_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float 2.000000e+00, float [[X]]) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @test_simplify3_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float 2.000000e+00, float [[X]]) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @test_simplify3_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[RETVAL:%.*]] = call float @powf(float 2.000000e+00, float [[X]]) #[[ATTR2:[0-9]+]]
+; VC64-NEXT:    ret float [[RETVAL]]
+;
+; VC83-LABEL: define float @test_simplify3_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[X]])
+; VC83-NEXT:    ret float [[EXP2]]
+;
+; NOLIB-LABEL: define float @test_simplify3_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float 2.000000e+00, float [[X]]) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 2.0, float %x) #0
+  ret float %retval
+}
+
+define double @test_simplify3n_noerrno(double %x) {
+; ANY-LABEL: define double @test_simplify3n_noerrno(
 ; ANY-SAME: double [[X:%.*]]) {
 ; ANY-NEXT:    [[MUL:%.*]] = fmul double [[X]], -2.000000e+00
-; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[MUL]])
+; ANY-NEXT:    [[EXP2:%.*]] = call double @llvm.exp2.f64(double [[MUL]])
 ; ANY-NEXT:    ret double [[EXP2]]
 ;
-; VC32-LABEL: define double @test_simplify3n(
+; VC32-LABEL: define double @test_simplify3n_noerrno(
 ; VC32-SAME: double [[X:%.*]]) {
-; VC32-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]])
+; VC32-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]]) #[[ATTR2]]
 ; VC32-NEXT:    ret double [[RETVAL]]
 ;
-; VC19-LABEL: define double @test_simplify3n(
+; VC19-LABEL: define double @test_simplify3n_noerrno(
 ; VC19-SAME: double [[X:%.*]]) {
 ; VC19-NEXT:    [[MUL:%.*]] = fmul double [[X]], -2.000000e+00
-; VC19-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[MUL]])
+; VC19-NEXT:    [[EXP2:%.*]] = call double @llvm.exp2.f64(double [[MUL]])
 ; VC19-NEXT:    ret double [[EXP2]]
 ;
-; VC64-LABEL: define double @test_simplify3n(
+; VC64-LABEL: define double @test_simplify3n_noerrno(
 ; VC64-SAME: double [[X:%.*]]) {
-; VC64-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]])
+; VC64-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]]) #[[ATTR2]]
 ; VC64-NEXT:    ret double [[RETVAL]]
 ;
-; NOLIB-LABEL: define double @test_simplify3n(
+; NOLIB-LABEL: define double @test_simplify3n_noerrno(
 ; NOLIB-SAME: double [[X:%.*]]) {
-; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]])
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.500000e-01, double [[X]]) #[[ATTR2]]
 ; NOLIB-NEXT:    ret double [[RETVAL]]
 ;
-  %retval = call double @pow(double 0.25, double %x)
+  %retval = call double @pow(double 0.25, double %x) #0
   ret double %retval
 }
 
@@ -256,6 +337,36 @@ define double @test_simplify4(double %x) {
   ret double %retval
 }
 
+define double @test_simplify4_noerrno(double %x) {
+; ANY-LABEL: define double @test_simplify4_noerrno(
+; ANY-SAME: double [[X:%.*]]) {
+; ANY-NEXT:    [[EXP2:%.*]] = call double @llvm.exp2.f64(double [[X]])
+; ANY-NEXT:    ret double [[EXP2]]
+;
+; VC32-LABEL: define double @test_simplify4_noerrno(
+; VC32-SAME: double [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.000000e+00, double [[X]]) #[[ATTR2]]
+; VC32-NEXT:    ret double [[RETVAL]]
+;
+; VC19-LABEL: define double @test_simplify4_noerrno(
+; VC19-SAME: double [[X:%.*]]) {
+; VC19-NEXT:    [[EXP2:%.*]] = call double @llvm.exp2.f64(double [[X]])
+; VC19-NEXT:    ret double [[EXP2]]
+;
+; VC64-LABEL: define double @test_simplify4_noerrno(
+; VC64-SAME: double [[X:%.*]]) {
+; VC64-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.000000e+00, double [[X]]) #[[ATTR2]]
+; VC64-NEXT:    ret double [[RETVAL]]
+;
+; NOLIB-LABEL: define double @test_simplify4_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double 2.000000e+00, double [[X]]) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double 2.0, double %x) #0
+  ret double %retval
+}
+
 define float @test_simplify4n(float %x) {
 ; ANY-LABEL: define float @test_simplify4n(
 ; ANY-SAME: float [[X:%.*]]) {
@@ -293,6 +404,43 @@ define float @test_simplify4n(float %x) {
   ret float %retval
 }
 
+define float @test_simplify4n_noerrno(float %x) {
+; ANY-LABEL: define float @test_simplify4n_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[MUL:%.*]] = fmul float [[X]], 3.000000e+00
+; ANY-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[MUL]])
+; ANY-NEXT:    ret float [[EXP2]]
+;
+; VC32-LABEL: define float @test_simplify4n_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float 8.000000e+00, float [[X]]) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @test_simplify4n_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float 8.000000e+00, float [[X]]) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @test_simplify4n_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[RETVAL:%.*]] = call float @powf(float 8.000000e+00, float [[X]]) #[[ATTR2]]
+; VC64-NEXT:    ret float [[RETVAL]]
+;
+; VC83-LABEL: define float @test_simplify4n_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[MUL:%.*]] = fmul float [[X]], 3.000000e+00
+; VC83-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[MUL]])
+; VC83-NEXT:    ret float [[EXP2]]
+;
+; NOLIB-LABEL: define float @test_simplify4n_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float 8.000000e+00, float [[X]]) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 8.0, float %x) #0
+  ret float %retval
+}
+
 ; TODO: should be able to simplify llvm.pow to llvm.exp2 even without libcalls
 define <2 x double> @test_simplify4v(<2 x double> %x) {
 ; ANY-LABEL: define <2 x double> @test_simplify4v(
@@ -390,6 +538,38 @@ define float @test_simplify5(float %x) {
   ret float %retval
 }
 
+define float @test_simplify5_noerrno(float %x) {
+; ANY-LABEL: define float @test_simplify5_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    ret float 1.000000e+00
+;
+; VC32-LABEL: define float @test_simplify5_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 0.000000e+00) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @test_simplify5_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 0.000000e+00) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @test_simplify5_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    ret float 1.000000e+00
+;
+; VC83-LABEL: define float @test_simplify5_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    ret float 1.000000e+00
+;
+; NOLIB-LABEL: define float @test_simplify5_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 0.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float %x, float 0.0) #0
+  ret float %retval
+}
+
 define <2 x float> @test_simplify5v(<2 x float> %x) {
 ; CHECK-LABEL: define <2 x float> @test_simplify5v(
 ; CHECK-SAME: <2 x float> [[X:%.*]]) {
@@ -413,6 +593,20 @@ define double @test_simplify6(double %x) {
   ret double %retval
 }
 
+define double @test_simplify6_noerrno(double %x) {
+; LIB-LABEL: define double @test_simplify6_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    ret double 1.000000e+00
+;
+; NOLIB-LABEL: define double @test_simplify6_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double [[X]], double 0.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double %x, double 0.0) #0
+  ret double %retval
+}
+
 define <2 x double> @test_simplify6v(<2 x double> %x) {
 ; CHECK-LABEL: define <2 x double> @test_simplify6v(
 ; CHECK-SAME: <2 x double> [[X:%.*]]) {
@@ -462,63 +656,101 @@ define float @powf_libcall_half_ninf(float %x) {
   ret float %retval
 }
 
+define float @powf_libcall_half_ninf_noerrno(float %x) {
+; ANY-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; ANY-NEXT:    [[ABS:%.*]] = call ninf float @llvm.fabs.f32(float [[SQRT]])
+; ANY-NEXT:    ret float [[ABS]]
+;
+; VC32-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; VC64-NEXT:    [[ABS:%.*]] = call ninf float @llvm.fabs.f32(float [[SQRT]])
+; VC64-NEXT:    ret float [[ABS]]
+;
+; VC83-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; VC83-NEXT:    [[ABS:%.*]] = call ninf float @llvm.fabs.f32(float [[SQRT]])
+; VC83-NEXT:    ret float [[ABS]]
+;
+; NOLIB-LABEL: define float @powf_libcall_half_ninf_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call ninf float @powf(float %x, float 0.5) #0
+  ret float %retval
+}
+
 ; Make sure assume works when inferring no infinities
-define float @powf_libcall_half_assume_ninf(float %x) {
-; ANY-LABEL: define float @powf_libcall_half_assume_ninf(
+define float @powf_libcall_half_assume_ninf_noerrno(float %x) {
+; ANY-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; ANY-SAME: float [[X:%.*]]) {
 ; ANY-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; ANY-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; ANY-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; ANY-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X]])
-; ANY-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; ANY-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[X]])
+; ANY-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
 ; ANY-NEXT:    ret float [[ABS]]
 ;
-; VC32-LABEL: define float @powf_libcall_half_assume_ninf(
+; VC32-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; VC32-SAME: float [[X:%.*]]) {
 ; VC32-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; VC32-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; VC32-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01)
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
 ; VC32-NEXT:    ret float [[RETVAL]]
 ;
-; VC51-LABEL: define float @powf_libcall_half_assume_ninf(
+; VC51-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; VC51-SAME: float [[X:%.*]]) {
 ; VC51-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; VC51-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; VC51-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01)
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
 ; VC51-NEXT:    ret float [[RETVAL]]
 ;
-; VC64-LABEL: define float @powf_libcall_half_assume_ninf(
+; VC64-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; VC64-SAME: float [[X:%.*]]) {
 ; VC64-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; VC64-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; VC64-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; VC64-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X]])
-; VC64-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC64-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[X]])
+; VC64-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
 ; VC64-NEXT:    ret float [[ABS]]
 ;
-; VC83-LABEL: define float @powf_libcall_half_assume_ninf(
+; VC83-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; VC83-SAME: float [[X:%.*]]) {
 ; VC83-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; VC83-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; VC83-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; VC83-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X]])
-; VC83-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC83-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[X]])
+; VC83-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
 ; VC83-NEXT:    ret float [[ABS]]
 ;
-; NOLIB-LABEL: define float @powf_libcall_half_assume_ninf(
+; NOLIB-LABEL: define float @powf_libcall_half_assume_ninf_noerrno(
 ; NOLIB-SAME: float [[X:%.*]]) {
 ; NOLIB-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; NOLIB-NEXT:    [[NOT_INF:%.*]] = fcmp one float [[FABS]], 0x7FF0000000000000
 ; NOLIB-NEXT:    call void @llvm.assume(i1 [[NOT_INF]])
-; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01)
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
 ; NOLIB-NEXT:    ret float [[RETVAL]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
   %not.inf = fcmp one float %fabs, 0x7FF0000000000000
   call void @llvm.assume(i1 %not.inf)
-  %retval = call float @powf(float %x, float 0.5)
+  %retval = call float @powf(float %x, float 0.5) #0
   ret float %retval
 }
 
@@ -560,6 +792,44 @@ define float @powf_libcall_half_ninf_tail(float %x) {
   ret float %retval
 }
 
+define float @powf_libcall_half_ninf_tail_noerrno(float %x) {
+; ANY-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; ANY-NEXT:    [[ABS:%.*]] = tail call ninf float @llvm.fabs.f32(float [[SQRT]])
+; ANY-NEXT:    ret float [[ABS]]
+;
+; VC32-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = tail call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = tail call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; VC64-NEXT:    [[ABS:%.*]] = tail call ninf float @llvm.fabs.f32(float [[SQRT]])
+; VC64-NEXT:    ret float [[ABS]]
+;
+; VC83-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float [[X]])
+; VC83-NEXT:    [[ABS:%.*]] = tail call ninf float @llvm.fabs.f32(float [[SQRT]])
+; VC83-NEXT:    ret float [[ABS]]
+;
+; NOLIB-LABEL: define float @powf_libcall_half_ninf_tail_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = tail call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = tail call ninf float @powf(float %x, float 0.5) #0
+  ret float %retval
+}
+
 define float @powf_libcall_half_ninf_musttail(float %x, float %y) {
 ; CHECK-LABEL: define float @powf_libcall_half_ninf_musttail(
 ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
@@ -570,6 +840,16 @@ define float @powf_libcall_half_ninf_musttail(float %x, float %y) {
   ret float %retval
 }
 
+define float @powf_libcall_half_ninf_musttail_noerrno(float %x, float %y) {
+; CHECK-LABEL: define float @powf_libcall_half_ninf_musttail_noerrno(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = musttail call ninf float @powf(float [[X]], float 5.000000e-01) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %retval = musttail call ninf float @powf(float %x, float 0.5) #0
+  ret float %retval
+}
+
 ; Check pow(x, 0.5) where x may be -infinity does not call a library sqrt function.
 
 define double @pow_libcall_half_no_FMF(double %x) {
@@ -582,6 +862,24 @@ define double @pow_libcall_half_no_FMF(double %x) {
   ret double %retval
 }
 
+define double @pow_libcall_half_no_FMF_noerrno(double %x) {
+; LIB-LABEL: define double @pow_libcall_half_no_FMF_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X]])
+; LIB-NEXT:    [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; LIB-NEXT:    [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; LIB-NEXT:    [[RETVAL:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; LIB-NEXT:    ret double [[RETVAL]]
+;
+; NOLIB-LABEL: define double @pow_libcall_half_no_FMF_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double [[X]], double 5.000000e-01) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double %x, double 0.5) #0
+  ret double %retval
+}
+
 ; Check pow(-infinity, 0.5) -> +infinity.
 
 define float @test_simplify9(float %x) {
@@ -636,6 +934,38 @@ define float @test_simplify11(float %x) {
   ret float %retval
 }
 
+define float @test_simplify11_noerrno(float %x) {
+; ANY-LABEL: define float @test_simplify11_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    ret float [[X]]
+;
+; VC32-LABEL: define float @test_simplify11_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 1.000000e+00) #[[ATTR2]]
+; VC32-NEXT:    ret float [[RETVAL]]
+;
+; VC51-LABEL: define float @test_simplify11_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 1.000000e+00) #[[ATTR2]]
+; VC51-NEXT:    ret float [[RETVAL]]
+;
+; VC64-LABEL: define float @test_simplify11_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    ret float [[X]]
+;
+; VC83-LABEL: define float @test_simplify11_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    ret float [[X]]
+;
+; NOLIB-LABEL: define float @test_simplify11_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call float @powf(float [[X]], float 1.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float %x, float 1.0) #0
+  ret float %retval
+}
+
 define <2 x float> @test_simplify11v(<2 x float> %x) {
 ; CHECK-LABEL: define <2 x float> @test_simplify11v(
 ; CHECK-SAME: <2 x float> [[X:%.*]]) {
@@ -659,6 +989,20 @@ define double @test_simplify12(double %x) {
   ret double %retval
 }
 
+define double @test_simplify12_noerrno(double %x) {
+; LIB-LABEL: define double @test_simplify12_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    ret double [[X]]
+;
+; NOLIB-LABEL: define double @test_simplify12_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[RETVAL:%.*]] = call double @pow(double [[X]], double 1.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double %x, double 1.0) #0
+  ret double %retval
+}
+
 define <2 x double> @test_simplify12v(<2 x double> %x) {
 ; CHECK-LABEL: define <2 x double> @test_simplify12v(
 ; CHECK-SAME: <2 x double> [[X:%.*]]) {
@@ -705,6 +1049,41 @@ define float @pow2_strict(float %x) {
   ret float %r
 }
 
+define float @pow2_strict_noerrno(float %x) {
+; ANY-LABEL: define float @pow2_strict_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul float [[X]], [[X]]
+; ANY-NEXT:    ret float [[SQUARE]]
+;
+; VC32-LABEL: define float @pow2_strict_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; VC32-NEXT:    ret float [[R]]
+;
+; VC51-LABEL: define float @pow2_strict_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; VC51-NEXT:    ret float [[R]]
+;
+; VC64-LABEL: define float @pow2_strict_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[SQUARE:%.*]] = fmul float [[X]], [[X]]
+; VC64-NEXT:    ret float [[SQUARE]]
+;
+; VC83-LABEL: define float @pow2_strict_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[SQUARE:%.*]] = fmul float [[X]], [[X]]
+; VC83-NEXT:    ret float [[SQUARE]]
+;
+; NOLIB-LABEL: define float @pow2_strict_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[R]]
+;
+  %r = call float @powf(float %x, float 2.0) #0
+  ret float %r
+}
+
 define <2 x float> @pow2_strictv(<2 x float> %x) {
 ; CHECK-LABEL: define <2 x float> @pow2_strictv(
 ; CHECK-SAME: <2 x float> [[X:%.*]]) {
@@ -730,6 +1109,21 @@ define double @pow2_double_strict(double %x) {
   ret double %r
 }
 
+define double @pow2_double_strict_noerrno(double %x) {
+; LIB-LABEL: define double @pow2_double_strict_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    [[SQUARE:%.*]] = fmul double [[X]], [[X]]
+; LIB-NEXT:    ret double [[SQUARE]]
+;
+; NOLIB-LABEL: define double @pow2_double_strict_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[R:%.*]] = call double @pow(double [[X]], double 2.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[R]]
+;
+  %r = call double @pow(double %x, double 2.0) #0
+  ret double %r
+}
+
 define <2 x double> @pow2_double_strictv(<2 x double> %x) {
 ; CHECK-LABEL: define <2 x double> @pow2_double_strictv(
 ; CHECK-SAME: <2 x double> [[X:%.*]]) {
@@ -777,6 +1171,41 @@ define float @pow2_fast(float %x) {
   ret float %r
 }
 
+define float @pow2_fast_noerrno(float %x) {
+; ANY-LABEL: define float @pow2_fast_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X]], [[X]]
+; ANY-NEXT:    ret float [[SQUARE]]
+;
+; VC32-LABEL: define float @pow2_fast_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[R:%.*]] = call fast float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; VC32-NEXT:    ret float [[R]]
+;
+; VC51-LABEL: define float @pow2_fast_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[R:%.*]] = call fast float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; VC51-NEXT:    ret float [[R]]
+;
+; VC64-LABEL: define float @pow2_fast_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X]], [[X]]
+; VC64-NEXT:    ret float [[SQUARE]]
+;
+; VC83-LABEL: define float @pow2_fast_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X]], [[X]]
+; VC83-NEXT:    ret float [[SQUARE]]
+;
+; NOLIB-LABEL: define float @pow2_fast_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[R:%.*]] = call fast float @powf(float [[X]], float 2.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[R]]
+;
+  %r = call fast float @powf(float %x, float 2.0) #0
+  ret float %r
+}
+
 ; Check pow(x, -1.0) -> 1.0/x.
 
 define float @pow_neg1_strict(float %x) {
@@ -814,6 +1243,41 @@ define float @pow_neg1_strict(float %x) {
   ret float %r
 }
 
+define float @pow_neg1_strict_noerrno(float %x) {
+; ANY-LABEL: define float @pow_neg1_strict_noerrno(
+; ANY-SAME: float [[X:%.*]]) {
+; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X]]
+; ANY-NEXT:    ret float [[RECIPROCAL]]
+;
+; VC32-LABEL: define float @pow_neg1_strict_noerrno(
+; VC32-SAME: float [[X:%.*]]) {
+; VC32-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float -1.000000e+00) #[[ATTR2]]
+; VC32-NEXT:    ret float [[R]]
+;
+; VC51-LABEL: define float @pow_neg1_strict_noerrno(
+; VC51-SAME: float [[X:%.*]]) {
+; VC51-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float -1.000000e+00) #[[ATTR2]]
+; VC51-NEXT:    ret float [[R]]
+;
+; VC64-LABEL: define float @pow_neg1_strict_noerrno(
+; VC64-SAME: float [[X:%.*]]) {
+; VC64-NEXT:    [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X]]
+; VC64-NEXT:    ret float [[RECIPROCAL]]
+;
+; VC83-LABEL: define float @pow_neg1_strict_noerrno(
+; VC83-SAME: float [[X:%.*]]) {
+; VC83-NEXT:    [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X]]
+; VC83-NEXT:    ret float [[RECIPROCAL]]
+;
+; NOLIB-LABEL: define float @pow_neg1_strict_noerrno(
+; NOLIB-SAME: float [[X:%.*]]) {
+; NOLIB-NEXT:    [[R:%.*]] = call float @powf(float [[X]], float -1.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret float [[R]]
+;
+  %r = call float @powf(float %x, float -1.0) #0
+  ret float %r
+}
+
 define <2 x float> @pow_neg1_strictv(<2 x float> %x) {
 ; CHECK-LABEL: define <2 x float> @pow_neg1_strictv(
 ; CHECK-SAME: <2 x float> [[X:%.*]]) {
@@ -839,6 +1303,21 @@ define double @pow_neg1_double_fast(double %x) {
   ret double %r
 }
 
+define double @pow_neg1_double_fast_noerrno(double %x) {
+; LIB-LABEL: define double @pow_neg1_double_fast_noerrno(
+; LIB-SAME: double [[X:%.*]]) {
+; LIB-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[X]]
+; LIB-NEXT:    ret double [[RECIPROCAL]]
+;
+; NOLIB-LABEL: define double @pow_neg1_double_fast_noerrno(
+; NOLIB-SAME: double [[X:%.*]]) {
+; NOLIB-NEXT:    [[R:%.*]] = call fast double @pow(double [[X]], double -1.000000e+00) #[[ATTR2]]
+; NOLIB-NEXT:    ret double [[R]]
+;
+  %r = call fast double @pow(double %x, double -1.0) #0
+  ret double %r
+}
+
 define <2 x double> @pow_neg1_double_fastv(<2 x double> %x) {
 ; CHECK-LABEL: define <2 x double> @pow_neg1_double_fastv(
 ; CHECK-SAME: <2 x double> [[X:%.*]]) {
@@ -879,6 +1358,21 @@ define float @test_simplify18(float %x) {
   ret float %retval
 }
 
+define float @test_simplify18_noerrno(float %x) {
+; CHECK-EXP10-LABEL: define float @test_simplify18_noerrno(
+; CHECK-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[EXP10:%.*]] = call float @llvm.exp10.f32(float [[X]])
+; CHECK-EXP10-NEXT:    ret float [[EXP10]]
+;
+; CHECK-NO-EXP10-LABEL: define float @test_simplify18_noerrno(
+; CHECK-NO-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X]]) #[[ATTR2]]
+; CHECK-NO-EXP10-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 10.0, float %x) #0
+  ret float %retval
+}
+
 define double @test_simplify19(double %x) {
 ; CHECK-EXP10-LABEL: define double @test_simplify19(
 ; CHECK-EXP10-SAME: double [[X:%.*]]) {
@@ -894,6 +1388,21 @@ define double @test_simplify19(double %x) {
   ret double %retval
 }
 
+define double @test_simplify19_noerrno(double %x) {
+; CHECK-EXP10-LABEL: define double @test_simplify19_noerrno(
+; CHECK-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[EXP10:%.*]] = call double @llvm.exp10.f64(double [[X]])
+; CHECK-EXP10-NEXT:    ret double [[EXP10]]
+;
+; CHECK-NO-EXP10-LABEL: define double @test_simplify19_noerrno(
+; CHECK-NO-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X]]) #[[ATTR2]]
+; CHECK-NO-EXP10-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double 10.0, double %x) #0
+  ret double %retval
+}
+
 define float @test_libcall_powf_10_f32_noerrno(float %x) {
 ; CHECK-EXP10-LABEL: define float @test_libcall_powf_10_f32_noerrno(
 ; CHECK-EXP10-SAME: float [[X:%.*]]) {
@@ -902,10 +1411,10 @@ define float @test_libcall_powf_10_f32_noerrno(float %x) {
 ;
 ; CHECK-NO-EXP10-LABEL: define float @test_libcall_powf_10_f32_noerrno(
 ; CHECK-NO-EXP10-SAME: float [[X:%.*]]) {
-; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X]]) #[[ATTR3:[0-9]+]]
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X]]) #[[ATTR2]]
 ; CHECK-NO-EXP10-NEXT:    ret float [[RETVAL]]
 ;
-  %retval = call float @powf(float 10.0, float %x) memory(none)
+  %retval = call float @powf(float 10.0, float %x) #0
   ret float %retval
 }
 
@@ -917,10 +1426,10 @@ define double @test_libcall_pow_10_f64_noerrno(double %x) {
 ;
 ; CHECK-NO-EXP10-LABEL: define double @test_libcall_pow_10_f64_noerrno(
 ; CHECK-NO-EXP10-SAME: double [[X:%.*]]) {
-; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X]]) #[[ATTR3]]
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X]]) #[[ATTR2]]
 ; CHECK-NO-EXP10-NEXT:    ret double [[RETVAL]]
 ;
-  %retval = call double @pow(double 10.0, double %x) memory(none)
+  %retval = call double @pow(double 10.0, double %x) #0
   ret double %retval
 }
 
@@ -1024,3 +1533,5 @@ define <2 x bfloat> @test_pow_10_v2bf16(<2 x bfloat> %x) {
   %retval = call <2 x bfloat> @llvm.pow.v2bf16(<2 x bfloat> <bfloat 10.0, bfloat 10.0>, <2 x bfloat> %x)
   ret <2 x bfloat> %retval
 }
+
+attributes #0 = { nounwind memory(none) }
diff --git a/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll b/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
index cd05022b0d35da079c07f4a43c8fd1e054f20313..e5906e7a96973017d3a32d24fe46a6b32a6f3a96 100644
--- a/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
+++ b/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
@@ -7,9 +7,9 @@
 
 define zeroext i1 @test1(i32 %lhs, i32 %rhs) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[LHS:%.*]], 5
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[XOR]], 10
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[XOR]], [[RHS:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LHS:%.*]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[LHS]], [[RHS:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[TMP1]], 5
 ; CHECK-NEXT:    [[SEL:%.*]] = or i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    ret i1 [[SEL]]
 ;
@@ -23,9 +23,9 @@ define zeroext i1 @test1(i32 %lhs, i32 %rhs) {
 
 define zeroext i1 @test1_logical(i32 %lhs, i32 %rhs) {
 ; CHECK-LABEL: @test1_logical(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[LHS:%.*]], 5
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[XOR]], 10
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[XOR]], [[RHS:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LHS:%.*]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[LHS]], [[RHS:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[TMP1]], 5
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], i1 true, i1 [[CMP2]]
 ; CHECK-NEXT:    ret i1 [[SEL]]
 ;
@@ -40,7 +40,7 @@ define zeroext i1 @test1_logical(i32 %lhs, i32 %rhs) {
 define zeroext i1 @test2(i32 %lhs, i32 %rhs) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[LHS:%.*]], [[RHS:%.*]]
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[XOR]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LHS]], [[RHS]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[XOR]], 32
 ; CHECK-NEXT:    [[SEL:%.*]] = xor i1 [[CMP1]], [[CMP2]]
 ; CHECK-NEXT:    ret i1 [[SEL]]
diff --git a/llvm/test/Transforms/InstCombine/ptrtoint-nullgep.ll b/llvm/test/Transforms/InstCombine/ptrtoint-nullgep.ll
index e6e63778d82f005091be03583f5d2d8ba9e3f842..9d6f0abce84c7c5aed4b43d74f92a859b7f67a42 100644
--- a/llvm/test/Transforms/InstCombine/ptrtoint-nullgep.ll
+++ b/llvm/test/Transforms/InstCombine/ptrtoint-nullgep.ll
@@ -68,10 +68,10 @@ define void @constant_fold_ptrtoint_of_gep_of_nullgep() {
 ; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 1234) to i64))
 ; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 1234) to i64))
 ; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 1234) to i64))
-; LLPARSER-NEXT:    call void @use_i64(i64 0)
-; LLPARSER-NEXT:    call void @use_i64(i64 0)
-; LLPARSER-NEXT:    call void @use_i64(i64 0)
-; LLPARSER-NEXT:    call void @use_i64(i64 0)
+; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; LLPARSER-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
 ; LLPARSER-NEXT:    ret void
 ;
 ; INSTSIMPLIFY-LABEL: define {{[^@]+}}@constant_fold_ptrtoint_of_gep_of_nullgep() {
@@ -83,10 +83,10 @@ define void @constant_fold_ptrtoint_of_gep_of_nullgep() {
 ; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 1234) to i64))
 ; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 1234) to i64))
 ; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 1234) to i64))
-; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 0)
-; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 0)
-; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 0)
-; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 0)
+; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
+; INSTSIMPLIFY-NEXT:    call void @use_i64(i64 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr (i8, ptr addrspace(1) null, i64 -1), i64 1) to i64))
 ; INSTSIMPLIFY-NEXT:    ret void
 ;
 ; INSTCOMBINE-LABEL: define {{[^@]+}}@constant_fold_ptrtoint_of_gep_of_nullgep() {
diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
index 729ca03ddfd15028c68dcdad148eac9d8090e1ce..e4dd2d10637d3ee5ed922440d169a4b6980a821b 100644
--- a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
+++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
@@ -125,7 +125,7 @@ define { i32, i1 } @fold_sub_simple(i32 %x) {
 
 define { i32, i1 } @fold_with_distjoin_or(i32 %x) {
 ; CHECK-LABEL: @fold_with_distjoin_or(
-; CHECK-NEXT:    [[B:%.*]] = add i32 [[X:%.*]], 6
+; CHECK-NEXT:    [[B:%.*]] = add nsw i32 [[X:%.*]], 6
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 poison, i1 false }, i32 [[B]], 0
 ; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index d3cf2af4be6529ff2bb665163795cef58a7b15d0..b37e9175b26a5f26767da2fbf853b777dff21705 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -667,7 +667,7 @@ define i1 @test39(i1 %cond, double %x) {
 ; CHECK-LABEL: @test39(
 ; CHECK-NEXT:    ret i1 true
 ;
-  %s = select i1 %cond, double %x, double 0x7FF0000000000000 ; RHS = +infty
+  %s = select i1 %cond, double %x, double 0x7FF0000000000000   ; RHS = +infty
   %cmp = fcmp ule double %x, %s
   ret i1 %cmp
 }
@@ -1364,7 +1364,7 @@ define i32 @PR23757_ne(i32 %x, ptr %p) {
 ; CHECK-NEXT:    ret i32 -2147483648
 ;
   %cmp = icmp ne i32 %x, 2147483647
-  store i1 %cmp, ptr %p ; thwart predicate canonicalization
+  store i1 %cmp, ptr %p   ; thwart predicate canonicalization
   %add = add nsw i32 %x, 1
   %sel = select i1 %cmp, i32 -2147483648, i32 %add
   ret i32 %sel
@@ -1378,7 +1378,7 @@ define i32 @PR23757_ne_swapped(i32 %x, ptr %p) {
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp ne i32 %x, 2147483647
-  store i1 %cmp, ptr %p ; thwart predicate canonicalization
+  store i1 %cmp, ptr %p   ; thwart predicate canonicalization
   %add = add nsw i32 %x, 1
   %sel = select i1 %cmp, i32 %add, i32 -2147483648
   ret i32 %sel
@@ -2787,16 +2787,67 @@ define <2 x i8> @select_replacement_add_eq_vec_nonuniform(<2 x i8> %x, <2 x i8>
 define <2 x i8> @select_replacement_add_eq_vec_poison(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @select_replacement_add_eq_vec_poison(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 poison>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 poison>, <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %cmp = icmp eq <2 x i8> %x, <i8 1, i8 poison>
+  %add = add <2 x i8> %x, <i8 1, i8 1>
+  %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y
+  ret <2 x i8> %sel
+}
+
+define <2 x i8> @select_replacement_add_eq_vec_undef(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @select_replacement_add_eq_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 undef>
 ; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], <i8 1, i8 1>
 ; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
-  %cmp = icmp eq <2 x i8> %x, <i8 1, i8 poison>
+  %cmp = icmp eq <2 x i8> %x, <i8 1, i8 undef>
   %add = add <2 x i8> %x, <i8 1, i8 1>
   %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y
   ret <2 x i8> %sel
 }
 
+define <2 x i8> @select_replacement_add_eq_vec_undef_okay(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 undef>, <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %cmp = icmp eq <2 x i8> %x, <i8 1, i8 1>
+  %add = add <2 x i8> %x, <i8 1, i8 undef>
+  %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y
+  ret <2 x i8> %sel
+}
+
+
+define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay_todo(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 undef>
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], <i8 1, i8 undef>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %cmp = icmp eq <2 x i8> %x, <i8 1, i8 undef>
+  %add = add <2 x i8> %x, <i8 1, i8 undef>
+  %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y
+  ret <2 x i8> %sel
+}
+
+define <2 x i8> @select_replacement_xor_eq_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_replacement_xor_eq_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> zeroinitializer, <2 x i8> [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %cmp = icmp eq <2 x i8> %x, %y
+  %add = xor <2 x i8> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %z
+  ret <2 x i8> %sel
+}
+
+
 define i8 @select_replacement_add_ne(i8 %x, i8 %y) {
 ; CHECK-LABEL: @select_replacement_add_ne(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X:%.*]], 1
@@ -2835,12 +2886,25 @@ define i8 @select_replacement_sub_noundef(i8 %x, i8 noundef %y, i8 %z) {
   ret i8 %sel
 }
 
+define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %yy, i8 %z) {
+; CHECK-LABEL: @select_replacement_sub_noundef_but_may_be_poison(
+; CHECK-NEXT:    [[Y:%.*]] = shl nuw i8 [[YY:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %y = shl nuw i8 %yy, 1
+  %cmp = icmp eq i8 %x, %y
+  %sub = sub i8 %x, %y
+  %sel = select i1 %cmp, i8 %sub, i8 %z
+  ret i8 %sel
+}
+
 ; TODO: The transform is also safe without noundef.
 define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) {
 ; CHECK-LABEL: @select_replacement_sub(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[Y]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[SUB]], i8 [[Z:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, %y
@@ -4483,7 +4547,7 @@ define i32 @src_no_trans_select_xor_eq0_xor_or(i32 %x, i32 %y) {
 define i32 @src_no_trans_select_xor_eq0_and_xor(i32 %x, i32 %y) {
 ; CHECK-LABEL: @src_no_trans_select_xor_eq0_and_xor(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], 0
+; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 [[AND]], i32 [[XOR]]
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -4499,7 +4563,7 @@ define i32 @src_no_trans_select_xor_eq0_and_xor(i32 %x, i32 %y) {
 define i32 @src_no_trans_select_xor_eq0_or_xor(i32 %x, i32 %y) {
 ; CHECK-LABEL: @src_no_trans_select_xor_eq0_or_xor(
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], 0
+; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 [[OR]], i32 [[XOR]]
 ; CHECK-NEXT:    ret i32 [[COND]]
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 8da52e074637363f6047e129877257e82c831b16..0700d7a62ee15c0de02d39e75d4d85cd741e2f37 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -2240,4 +2240,84 @@ define i129 @shift_zext_not_nneg(i8 %arg) {
   ret i129 %shl
 }
 
+define i8 @src_shl_nsw(i8 %x) {
+; CHECK-LABEL: @src_shl_nsw(
+; CHECK-NEXT:    [[R:%.*]] = shl nsw i8 32, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = shl nsw i8 1, %x
+  %r = shl nsw i8 %sh, 5
+  ret i8 %r
+}
+
+define i8 @src_shl_nsw_fail(i8 %x) {
+; CHECK-LABEL: @src_shl_nsw_fail(
+; CHECK-NEXT:    [[R:%.*]] = shl i8 32, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = shl nsw i8 1, %x
+  %r = shl i8 %sh, 5
+  ret i8 %r
+}
+
+define i8 @src_shl_nuw(i8 %x) {
+; CHECK-LABEL: @src_shl_nuw(
+; CHECK-NEXT:    [[R:%.*]] = shl nuw i8 12, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = shl nuw i8 3, %x
+  %r = shl nuw i8 %sh, 2
+  ret i8 %r
+}
+
+define i8 @src_shl_nuw_fail(i8 %x) {
+; CHECK-LABEL: @src_shl_nuw_fail(
+; CHECK-NEXT:    [[R:%.*]] = shl i8 12, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = shl i8 3, %x
+  %r = shl nuw i8 %sh, 2
+  ret i8 %r
+}
+
+define i8 @src_lshr_exact(i8 %x) {
+; CHECK-LABEL: @src_lshr_exact(
+; CHECK-NEXT:    [[R:%.*]] = lshr exact i8 48, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = lshr exact i8 96, %x
+  %r = lshr exact i8 %sh, 1
+  ret i8 %r
+}
+
+define i8 @src_lshr_exact_fail(i8 %x) {
+; CHECK-LABEL: @src_lshr_exact_fail(
+; CHECK-NEXT:    [[R:%.*]] = lshr i8 48, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = lshr exact i8 96, %x
+  %r = lshr i8 %sh, 1
+  ret i8 %r
+}
+
+define i8 @src_ashr_exact(i8 %x) {
+; CHECK-LABEL: @src_ashr_exact(
+; CHECK-NEXT:    [[R:%.*]] = ashr exact i8 -8, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = ashr exact i8 -32, %x
+  %r = ashr exact i8 %sh, 2
+  ret i8 %r
+}
+
+define i8 @src_ashr_exact_fail(i8 %x) {
+; CHECK-LABEL: @src_ashr_exact_fail(
+; CHECK-NEXT:    [[R:%.*]] = ashr i8 -8, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = ashr i8 -32, %x
+  %r = ashr exact i8 %sh, 2
+  ret i8 %r
+}
+
 declare i16 @llvm.umax.i16(i16, i16)
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
index 27ab11446b571948130bc62cdf6c3b2faac8e8ac..39037761c81bb3c4d51513e6ced6d0e6a7c2b341 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
@@ -1,10 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -aarch64-lit -aarch64-lit-verify -verify-dom-info -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s
-; RUN: opt -aarch64-lit -simplifycfg -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=LOOP-DEL
-; RUN: opt -aarch64-lit -mtriple aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=NO-TRANSFORM
-; RUN: opt -p aarch64-lit -aarch64-lit-verify -verify-dom-info -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s
-; RUN: opt -passes='function(loop(aarch64-lit)),simplifycfg' -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=LOOP-DEL
-; RUN: opt -p aarch64-lit -mtriple aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=NO-TRANSFORM
+; RUN: opt -p loop-idiom-vectorize -loop-idiom-vectorize-verify -verify-dom-info -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+; RUN: opt -passes='function(loop(loop-idiom-vectorize)),simplifycfg' -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=LOOP-DEL
+; RUN: opt -p loop-idiom-vectorize -mtriple aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=NO-TRANSFORM
 
 define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; CHECK-LABEL: define i32 @compare_bytes_simple(
@@ -33,36 +30,36 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       mismatch_sve_loop_preheader:
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       mismatch_vec_loop_preheader:
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; CHECK-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; CHECK:       mismatch_sve_loop:
-; CHECK-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; CHECK:       mismatch_vec_loop:
+; CHECK-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; CHECK:       mismatch_sve_loop_inc:
-; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; CHECK:       mismatch_vec_loop_inc:
+; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[MISMATCH_END:%.*]]
-; CHECK:       mismatch_sve_loop_found:
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[MISMATCH_END:%.*]]
+; CHECK:       mismatch_vec_loop_found:
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; CHECK-NEXT:    br label [[MISMATCH_END]]
 ; CHECK:       mismatch_loop_pre:
@@ -81,7 +78,7 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MISMATCH_END]], label [[MISMATCH_LOOP]]
 ; CHECK:       mismatch_end:
-; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; CHECK-NEXT:    br i1 true, label [[BYTE_COMPARE:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       while.cond:
 ; CHECK-NEXT:    [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ]
@@ -128,36 +125,36 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; LOOP-DEL-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; LOOP-DEL-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; LOOP-DEL-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1:![0-9]+]]
-; LOOP-DEL:       mismatch_sve_loop_preheader:
+; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1:![0-9]+]]
+; LOOP-DEL:       mismatch_vec_loop_preheader:
 ; LOOP-DEL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; LOOP-DEL-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; LOOP-DEL-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; LOOP-DEL:       mismatch_sve_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; LOOP-DEL:       mismatch_vec_loop:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; LOOP-DEL-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; LOOP-DEL-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; LOOP-DEL:       mismatch_sve_loop_inc:
-; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; LOOP-DEL:       mismatch_vec_loop_inc:
+; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; LOOP-DEL-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[WHILE_END:%.*]]
-; LOOP-DEL:       mismatch_sve_loop_found:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[WHILE_END:%.*]]
+; LOOP-DEL:       mismatch_vec_loop_found:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; LOOP-DEL-NEXT:    br label [[WHILE_END]]
 ; LOOP-DEL:       mismatch_loop_pre:
@@ -176,7 +173,7 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; LOOP-DEL-NEXT:    [[RES:%.*]] = add i32 [[MISMATCH_RESULT]], [[EXTRA]]
 ; LOOP-DEL-NEXT:    ret i32 [[RES]]
 ;
@@ -256,36 +253,36 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; CHECK:       mismatch_sve_loop_preheader:
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; CHECK:       mismatch_vec_loop_preheader:
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; CHECK-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; CHECK:       mismatch_sve_loop:
-; CHECK-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; CHECK:       mismatch_vec_loop:
+; CHECK-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; CHECK:       mismatch_sve_loop_inc:
-; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; CHECK:       mismatch_vec_loop_inc:
+; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[MISMATCH_END:%.*]]
-; CHECK:       mismatch_sve_loop_found:
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[MISMATCH_END:%.*]]
+; CHECK:       mismatch_vec_loop_found:
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; CHECK-NEXT:    br label [[MISMATCH_END]]
 ; CHECK:       mismatch_loop_pre:
@@ -304,7 +301,7 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MISMATCH_END]], label [[MISMATCH_LOOP]]
 ; CHECK:       mismatch_end:
-; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; CHECK-NEXT:    br i1 true, label [[BYTE_COMPARE:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       while.cond:
 ; CHECK-NEXT:    [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ]
@@ -349,36 +346,36 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; LOOP-DEL-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; LOOP-DEL-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; LOOP-DEL-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; LOOP-DEL:       mismatch_sve_loop_preheader:
+; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; LOOP-DEL:       mismatch_vec_loop_preheader:
 ; LOOP-DEL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; LOOP-DEL-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; LOOP-DEL-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; LOOP-DEL:       mismatch_sve_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; LOOP-DEL:       mismatch_vec_loop:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; LOOP-DEL-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; LOOP-DEL-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; LOOP-DEL:       mismatch_sve_loop_inc:
-; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; LOOP-DEL:       mismatch_vec_loop_inc:
+; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; LOOP-DEL-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[WHILE_END:%.*]]
-; LOOP-DEL:       mismatch_sve_loop_found:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[WHILE_END:%.*]]
+; LOOP-DEL:       mismatch_vec_loop_found:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; LOOP-DEL-NEXT:    br label [[WHILE_END]]
 ; LOOP-DEL:       mismatch_loop_pre:
@@ -397,7 +394,7 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[MISMATCH_RESULT]]
 ;
 ; NO-TRANSFORM-LABEL: define i32 @compare_bytes_signed_wrap(
@@ -472,36 +469,36 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; CHECK:       mismatch_sve_loop_preheader:
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; CHECK:       mismatch_vec_loop_preheader:
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; CHECK-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; CHECK:       mismatch_sve_loop:
-; CHECK-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; CHECK:       mismatch_vec_loop:
+; CHECK-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; CHECK:       mismatch_sve_loop_inc:
-; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; CHECK:       mismatch_vec_loop_inc:
+; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[MISMATCH_END:%.*]]
-; CHECK:       mismatch_sve_loop_found:
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[MISMATCH_END:%.*]]
+; CHECK:       mismatch_vec_loop_found:
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; CHECK-NEXT:    br label [[MISMATCH_END]]
 ; CHECK:       mismatch_loop_pre:
@@ -520,7 +517,7 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MISMATCH_END]], label [[MISMATCH_LOOP]]
 ; CHECK:       mismatch_end:
-; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; CHECK-NEXT:    br i1 true, label [[BYTE_COMPARE:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       while.cond:
 ; CHECK-NEXT:    [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ]
@@ -576,36 +573,36 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; LOOP-DEL-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; LOOP-DEL-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; LOOP-DEL-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; LOOP-DEL:       mismatch_sve_loop_preheader:
+; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; LOOP-DEL:       mismatch_vec_loop_preheader:
 ; LOOP-DEL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; LOOP-DEL-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; LOOP-DEL-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; LOOP-DEL:       mismatch_sve_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; LOOP-DEL:       mismatch_vec_loop:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; LOOP-DEL-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; LOOP-DEL-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; LOOP-DEL:       mismatch_sve_loop_inc:
-; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; LOOP-DEL:       mismatch_vec_loop_inc:
+; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; LOOP-DEL-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[BYTE_COMPARE:%.*]]
-; LOOP-DEL:       mismatch_sve_loop_found:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[BYTE_COMPARE:%.*]]
+; LOOP-DEL:       mismatch_vec_loop_found:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; LOOP-DEL-NEXT:    br label [[BYTE_COMPARE]]
 ; LOOP-DEL:       mismatch_loop_pre:
@@ -624,7 +621,7 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[BYTE_COMPARE]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       byte.compare:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; LOOP-DEL-NEXT:    [[TMP45:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP45]], i32 [[N]], i32 [[MISMATCH_RESULT]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT4:%.*]] = select i1 [[TMP45]], ptr [[D]], ptr [[C]]
@@ -729,36 +726,36 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; CHECK:       mismatch_sve_loop_preheader:
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; CHECK:       mismatch_vec_loop_preheader:
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; CHECK-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; CHECK:       mismatch_sve_loop:
-; CHECK-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; CHECK:       mismatch_vec_loop:
+; CHECK-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; CHECK:       mismatch_sve_loop_inc:
-; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; CHECK:       mismatch_vec_loop_inc:
+; CHECK-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[MISMATCH_END:%.*]]
-; CHECK:       mismatch_sve_loop_found:
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; CHECK-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[MISMATCH_END:%.*]]
+; CHECK:       mismatch_vec_loop_found:
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; CHECK-NEXT:    br label [[MISMATCH_END]]
 ; CHECK:       mismatch_loop_pre:
@@ -777,7 +774,7 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MISMATCH_END]], label [[MISMATCH_LOOP]]
 ; CHECK:       mismatch_end:
-; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; CHECK-NEXT:    br i1 true, label [[BYTE_COMPARE:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       while.cond:
 ; CHECK-NEXT:    [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ]
@@ -828,36 +825,36 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; LOOP-DEL-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
 ; LOOP-DEL-NEXT:    [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
 ; LOOP-DEL-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; LOOP-DEL:       mismatch_sve_loop_preheader:
+; LOOP-DEL-NEXT:    br i1 [[TMP18]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; LOOP-DEL:       mismatch_vec_loop_preheader:
 ; LOOP-DEL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; LOOP-DEL-NEXT:    [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; LOOP-DEL-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; LOOP-DEL:       mismatch_sve_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_SVE_INDEX]]
-; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; LOOP-DEL:       mismatch_vec_loop:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; LOOP-DEL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; LOOP-DEL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; LOOP-DEL-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP24]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; LOOP-DEL-NEXT:    [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; LOOP-DEL-NEXT:    [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
 ; LOOP-DEL-NEXT:    [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; LOOP-DEL:       mismatch_sve_loop_inc:
-; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP21]]
+; LOOP-DEL-NEXT:    br i1 [[TMP28]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; LOOP-DEL:       mismatch_vec_loop_inc:
+; LOOP-DEL-NEXT:    [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
 ; LOOP-DEL-NEXT:    [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_SVE_LOOP]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_sve_loop_found:
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; LOOP-DEL-NEXT:    br i1 [[TMP31]], label [[MISMATCH_VEC_LOOP]], label [[WHILE_END]]
+; LOOP-DEL:       mismatch_vec_loop_found:
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; LOOP-DEL-NEXT:    [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP34]]
+; LOOP-DEL-NEXT:    [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
 ; LOOP-DEL-NEXT:    br label [[WHILE_END]]
 ; LOOP-DEL:       mismatch_loop_pre:
@@ -876,7 +873,7 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
 ; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[INC_LCSSA]]
 ;
 ; NO-TRANSFORM-LABEL: define i32 @compare_bytes_extra_cmp(
@@ -960,36 +957,36 @@ define void @compare_bytes_cleanup_block(ptr %src1, ptr %src2) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_SVE_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
-; CHECK:       mismatch_sve_loop_preheader:
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MISMATCH_LOOP_PRE]], label [[MISMATCH_VEC_LOOP_PREHEADER:%.*]], !prof [[PROF1]]
+; CHECK:       mismatch_vec_loop_preheader:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 1, i64 0)
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul nuw nsw i64 [[TMP16]], 16
-; CHECK-NEXT:    br label [[MISMATCH_SVE_LOOP:%.*]]
-; CHECK:       mismatch_sve_loop:
-; CHECK-NEXT:    [[MISMATCH_SVE_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP15]], [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP26:%.*]], [[MISMATCH_SVE_LOOP_INC:%.*]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_INDEX:%.*]] = phi i64 [ 1, [[MISMATCH_SVE_LOOP_PREHEADER]] ], [ [[TMP25:%.*]], [[MISMATCH_SVE_LOOP_INC]] ]
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[SRC1]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP18]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[MISMATCH_SVE_INDEX]]
-; CHECK-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP20]], i32 1, <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    br label [[MISMATCH_VEC_LOOP:%.*]]
+; CHECK:       mismatch_vec_loop:
+; CHECK-NEXT:    [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP15]], [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP26:%.*]], [[MISMATCH_VEC_LOOP_INC:%.*]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ 1, [[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP25:%.*]], [[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[SRC1]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP18]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP20]], i32 1, <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp ne <vscale x 16 x i8> [[TMP19]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = select <vscale x 16 x i1> [[MISMATCH_SVE_LOOP_PRED]], <vscale x 16 x i1> [[TMP22]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP22]], <vscale x 16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP23]])
-; CHECK-NEXT:    br i1 [[TMP24]], label [[MISMATCH_SVE_LOOP_FOUND:%.*]], label [[MISMATCH_SVE_LOOP_INC]]
-; CHECK:       mismatch_sve_loop_inc:
-; CHECK-NEXT:    [[TMP25]] = add nuw nsw i64 [[MISMATCH_SVE_INDEX]], [[TMP17]]
+; CHECK-NEXT:    br i1 [[TMP24]], label [[MISMATCH_VEC_LOOP_FOUND:%.*]], label [[MISMATCH_VEC_LOOP_INC]]
+; CHECK:       mismatch_vec_loop_inc:
+; CHECK-NEXT:    [[TMP25]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP26]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP25]], i64 0)
 ; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <vscale x 16 x i1> [[TMP26]], i64 0
-; CHECK-NEXT:    br i1 [[TMP27]], label [[MISMATCH_SVE_LOOP]], label [[MISMATCH_END:%.*]]
-; CHECK:       mismatch_sve_loop_found:
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP23]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_SVE_LOOP_PRED]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[MISMATCH_SVE_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_SVE_INDEX]], [[MISMATCH_SVE_LOOP]] ]
-; CHECK-NEXT:    [[TMP28:%.*]] = and <vscale x 16 x i1> [[MISMATCH_SVE_LAST_LOOP_PRED]], [[MISMATCH_SVE_FOUND_PRED]]
+; CHECK-NEXT:    br i1 [[TMP27]], label [[MISMATCH_VEC_LOOP]], label [[MISMATCH_END:%.*]]
+; CHECK:       mismatch_vec_loop_found:
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP23]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], [[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP28]], i1 true)
 ; CHECK-NEXT:    [[TMP30:%.*]] = zext i32 [[TMP29]] to i64
-; CHECK-NEXT:    [[TMP31:%.*]] = add nuw nsw i64 [[MISMATCH_SVE_FOUND_INDEX]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP30]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32
 ; CHECK-NEXT:    br label [[MISMATCH_END]]
 ; CHECK:       mismatch_loop_pre:
@@ -1008,7 +1005,7 @@ define void @compare_bytes_cleanup_block(ptr %src1, ptr %src2) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i32 [[TMP39]], 0
 ; CHECK-NEXT:    br i1 [[TMP40]], label [[MISMATCH_END]], label [[MISMATCH_LOOP]]
 ; CHECK:       mismatch_end:
-; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ 0, [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ 0, [[MISMATCH_SVE_LOOP_INC]] ], [ [[TMP32]], [[MISMATCH_SVE_LOOP_FOUND]] ]
+; CHECK-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ 0, [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ 0, [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP32]], [[MISMATCH_VEC_LOOP_FOUND]] ]
 ; CHECK-NEXT:    br i1 true, label [[BYTE_COMPARE:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       while.cond:
 ; CHECK-NEXT:    [[LEN:%.*]] = phi i32 [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ], [ 0, [[MISMATCH_END]] ]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
index 323e242620e66f4a85a228649a0de5b3394be369..64e8a6be998eaf491ab2eedcc1b9c35cb44af99a 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
@@ -2,7 +2,7 @@
 
 declare void @foo(i64)
 
-; Verify that redundant adds aren't inserted by LSR.
+; Verify that redundant adds or geps aren't inserted by LSR.
 ; CHECK-LABEL: @bar(
 define void @bar(ptr %A) {
 entry:
@@ -10,9 +10,11 @@ entry:
 
 while.cond:
 ; CHECK-LABEL: while.cond:
-; CHECK: add i64 %lsr.iv, 1
 ; CHECK-NOT: add i64 %lsr.iv, 1
 ; CHECK-LABEL: land.rhs:
+; CHECK: getelementptr i8, ptr %lsr.iv, i64 -8
+; CHECK-NOT: getelementptr i8, ptr %lsr.iv, i64 -8
+; CHECK-NOT: add i64, %lsr.iv, 1
   %indvars.iv28 = phi i64 [ %indvars.iv.next29, %land.rhs ], [ 50, %entry ]
   %cmp = icmp sgt i64 %indvars.iv28, 0
   br i1 %cmp, label %land.rhs, label %while.end
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4914bb72d89452afd244e87eeb0165459fb93d07
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+target triple = "riscv64"
+
+; This test was added as example motivation for the changes in #89927, which
+; causes LSR to drop solutions if deemed to be less profitable than the
+; starting point. At the time of adding this test, LSR's search heuristics
+; best identified solution was an unprofitable one. This could of course
+; change with future LSR improvements.
+
+%struct = type { i64, i32, i32, i32, i32, i32, i32, i32, i32, i64, i32, i64, i64, i32, i64 }
+
+define i32 @main() {
+; CHECK-LABEL: define i32 @main() {
+; CHECK-NEXT:    [[CALL:%.*]] = tail call ptr null(i64 0)
+; CHECK-NEXT:    br label %[[BB2:.*]]
+; CHECK:       [[BB1:.*:]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[CALL]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[LSR_IV30:%.*]] = phi i64 [ [[LSR_IV_NEXT31:%.*]], %[[BB2]] ], [ 8, [[BB:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV27:%.*]] = phi i64 [ [[LSR_IV_NEXT28:%.*]], %[[BB2]] ], [ 12, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV24:%.*]] = phi i64 [ [[LSR_IV_NEXT25:%.*]], %[[BB2]] ], [ 16, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV21:%.*]] = phi i64 [ [[LSR_IV_NEXT22:%.*]], %[[BB2]] ], [ 20, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV18:%.*]] = phi i64 [ [[LSR_IV_NEXT19:%.*]], %[[BB2]] ], [ 24, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV15:%.*]] = phi i64 [ [[LSR_IV_NEXT16:%.*]], %[[BB2]] ], [ 28, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV12:%.*]] = phi i64 [ [[LSR_IV_NEXT13:%.*]], %[[BB2]] ], [ 32, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV9:%.*]] = phi i64 [ [[LSR_IV_NEXT10:%.*]], %[[BB2]] ], [ 36, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV4:%.*]] = phi i64 [ [[LSR_IV_NEXT5:%.*]], %[[BB2]] ], [ 40, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], %[[BB2]] ], [ 48, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[BB2]] ], [ 72, [[BB]] ]
+; CHECK-NEXT:    [[SCEVGEP32:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV30]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP32]], align 8
+; CHECK-NEXT:    [[SCEVGEP29:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV27]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP29]], align 4
+; CHECK-NEXT:    [[SCEVGEP26:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV24]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP26]], align 8
+; CHECK-NEXT:    [[SCEVGEP23:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV21]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP23]], align 4
+; CHECK-NEXT:    [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV18]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP20]], align 8
+; CHECK-NEXT:    [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV15]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP17]], align 4
+; CHECK-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV12]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP14]], align 8
+; CHECK-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV9]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP11]], align 4
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV4]]
+; CHECK-NEXT:    store i64 0, ptr [[SCEVGEP6]], align 8
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV1]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP3]], align 8
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV]]
+; CHECK-NEXT:    store i32 0, ptr [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV4]]
+; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[SCEVGEP7]], i64 40
+; CHECK-NEXT:    store i64 0, ptr [[SCEVGEP8]], align 8
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add i64 [[LSR_IV1]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT5]] = add i64 [[LSR_IV4]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT10]] = add i64 [[LSR_IV9]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT13]] = add i64 [[LSR_IV12]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT16]] = add i64 [[LSR_IV15]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT19]] = add i64 [[LSR_IV18]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT22]] = add i64 [[LSR_IV21]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT25]] = add i64 [[LSR_IV24]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT28]] = add i64 [[LSR_IV27]], 88
+; CHECK-NEXT:    [[LSR_IV_NEXT31]] = add i64 [[LSR_IV30]], 88
+; CHECK-NEXT:    br label %[[BB2]]
+;
+0:
+  %call = tail call ptr null(i64 0)
+  br label %2
+
+1:
+  %load = load i32, ptr %call, align 4
+  ret i32 0
+
+2:
+  %phi = phi i64 [ 0, %0 ], [ %add, %2 ]
+  %getelementptr = getelementptr %struct, ptr %call, i64 %phi
+  %getelementptr3 = getelementptr i8, ptr %getelementptr, i64 8
+  store i32 0, ptr %getelementptr3, align 8
+  %getelementptr4 = getelementptr i8, ptr %getelementptr, i64 12
+  store i32 0, ptr %getelementptr4, align 4
+  %getelementptr5 = getelementptr i8, ptr %getelementptr, i64 16
+  store i32 0, ptr %getelementptr5, align 8
+  %getelementptr6 = getelementptr i8, ptr %getelementptr, i64 20
+  store i32 0, ptr %getelementptr6, align 4
+  %getelementptr7 = getelementptr i8, ptr %getelementptr, i64 24
+  store i32 0, ptr %getelementptr7, align 8
+  %getelementptr8 = getelementptr i8, ptr %getelementptr, i64 28
+  store i32 0, ptr %getelementptr8, align 4
+  %getelementptr9 = getelementptr i8, ptr %getelementptr, i64 32
+  store i32 0, ptr %getelementptr9, align 8
+  %getelementptr10 = getelementptr i8, ptr %getelementptr, i64 36
+  store i32 0, ptr %getelementptr10, align 4
+  %getelementptr11 = getelementptr i8, ptr %getelementptr, i64 40
+  store i64 0, ptr %getelementptr11, align 8
+  %getelementptr12 = getelementptr i8, ptr %getelementptr, i64 48
+  store i32 0, ptr %getelementptr12, align 8
+  %getelementptr13 = getelementptr i8, ptr %getelementptr, i64 72
+  store i32 0, ptr %getelementptr13, align 8
+  %getelementptr14 = getelementptr i8, ptr %getelementptr, i64 80
+  store i64 0, ptr %getelementptr14, align 8
+  %add = add i64 %phi, 1
+  br label %2
+}
diff --git a/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7fd4eb18f16eb88e407b20bde2b23420ecd8caa3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll
@@ -0,0 +1,562 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime -unroll-allow-partial -S | FileCheck %s
+
+declare void @f() convergent
+declare void @g()
+
+; Although this loop contains a convergent instruction, it should be
+; fully unrolled.
+define i32 @full_unroll() {
+; CHECK-LABEL: @full_unroll(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    br label [[L3:%.*]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       a:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_1:%.*]]
+; CHECK:       a.1:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_2:%.*]]
+; CHECK:       a.2:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 3
+  br label %a
+
+a:
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction, but it should be partially
+; unrolled.  The unroll count is the largest power of 2 that divides the
+; multiple -- 4, in this case.
+define i32 @runtime_unroll(i32 %n) {
+; CHECK-LABEL: @runtime_unroll(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 12
+; CHECK-NEXT:    br label [[L3:%.*]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_3:%.*]], [[A_3:%.*]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       a:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_1:%.*]]
+; CHECK:       a.1:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_2:%.*]]
+; CHECK:       a.2:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_3]]
+; CHECK:       a.3:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[INC_3]] = add nsw i32 [[X_0]], 4
+; CHECK-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i32 [[INC_3]], [[LOOP_CTL]]
+; CHECK-NEXT:    br i1 [[EXITCOND_3]], label [[EXIT:%.*]], label [[L3]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  %loop_ctl = mul nsw i32 %n, 12
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  br label %a
+
+a:
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction, so its partial unroll
+; count must divide its trip multiple.  This overrides its unroll
+; pragma -- we unroll exactly 8 times, even though 16 is requested.
+define i32 @pragma_unroll(i32 %n) {
+; CHECK-LABEL: @pragma_unroll(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 24
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_7:%.*]], [[A_7:%.*]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       a:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_1:%.*]]
+; CHECK:       a.1:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_2:%.*]]
+; CHECK:       a.2:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_3:%.*]]
+; CHECK:       a.3:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_4:%.*]]
+; CHECK:       a.4:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_5:%.*]]
+; CHECK:       a.5:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_6:%.*]]
+; CHECK:       a.6:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    br label [[A_7]]
+; CHECK:       a.7:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[INC_7]] = add nsw i32 [[X_0]], 8
+; CHECK-NEXT:    [[EXITCOND_7:%.*]] = icmp eq i32 [[INC_7]], [[LOOP_CTL]]
+; CHECK-NEXT:    br i1 [[EXITCOND_7]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  %loop_ctl = mul nsw i32 %n, 24
+  br label %l3, !llvm.loop !0
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  br label %a
+
+a:
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !0
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 divides trip count 4. The loop unroll should respect the pragma.
+define void @pragma_unroll_divisible_trip_count() {
+; CHECK-LABEL: @pragma_unroll_divisible_trip_count(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_1:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[INC_1]] = add nuw nsw i32 [[X_0]], 2
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], 4
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 4
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 divides trip multiple 2. The loop unroll should respect the pragma.
+define i32 @pragma_unroll_divisible_trip_multiple(i32 %n) {
+; CHECK-LABEL: @pragma_unroll_divisible_trip_multiple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 2
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_1:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[INC_1]] = add nsw i32 [[X_0]], 2
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], [[LOOP_CTL]]
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  %loop_ctl = mul nsw i32 %n, 2
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 is unknown to divide runtime trip count, the loop is not unrolled
+; since remainder is forbidden for unrolling convergent loop.
+define i32 @pragma_unroll_indivisible_runtime_trip_count(i32 %n) {
+; CHECK-LABEL: @pragma_unroll_indivisible_runtime_trip_count(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[X_0]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP4]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 does not divide trip count 5, the loop is not unrolled by 2
+; since remainder is forbidden for unrolling convergent loop. Instead, the
+; loop gets fully unrolled.
+define i32 @pragma_unroll_indivisible_trip_count() {
+; CHECK-LABEL: @pragma_unroll_indivisible_trip_count(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %anchor = call token @llvm.experimental.convergence.anchor()
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ]
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 5
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction that is anchored inside the loop
+; itself. It is unrolled by 2 with remainder, as requested by the loop metadata.
+define i32 @pragma_unroll_with_remainder(i32 %n) {
+; CHECK-LABEL: @pragma_unroll_with_remainder(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[N:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], -1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
+; CHECK:       entry.new:
+; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]]
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[INC_1:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    [[TOK_LOOP_1:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP_1]]) ]
+; CHECK-NEXT:    [[INC_1]] = add nsw i32 [[X_0]], 2
+; CHECK-NEXT:    [[NITER_NEXT_1]] = add i32 [[NITER]], 2
+; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i32 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT:    br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[L3]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       exit.unr-lcssa.loopexit:
+; CHECK-NEXT:    br label [[EXIT_UNR_LCSSA]]
+; CHECK:       exit.unr-lcssa:
+; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; CHECK-NEXT:    br i1 [[LCMP_MOD]], label [[L3_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       l3.epil.preheader:
+; CHECK-NEXT:    br label [[L3_EPIL:%.*]]
+; CHECK:       l3.epil:
+; CHECK-NEXT:    [[TOK_LOOP_EPIL:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP_EPIL]]) ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.anchor()
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; Don't unroll a loop that is extended by convergence controls.
+;
+; We could theoretically duplicate the extension part, but this is not
+; implemented.
+define i32 @extended_loop(i32 %n) {
+; CHECK-LABEL: @extended_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[L3]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[X_0]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP4]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+  %tok.loop = call token @llvm.experimental.convergence.anchor()
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  call void @f() [ "convergencectrl"(token %tok.loop) ]
+  ret i32 0
+}
+
+; Inner loop is extended beyond the outer loop. No unrolling possible.
+
+define i32 @extended_inner_loop_1(i32 %n, i1 %cond) {
+; CHECK-LABEL: @extended_inner_loop_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L3:%.*]]
+; CHECK:       l3:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[X_0]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 4
+; CHECK-NEXT:    br label [[L2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2]]) ]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[L2]], label [[LATCH]], !llvm.loop [[LOOP4]]
+; CHECK:       latch:
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2]]) ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  %tok.loop = call token @llvm.experimental.convergence.anchor()
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 4
+  br label %l2, !llvm.loop !1
+
+l2:
+  %tok.l2 = call token @llvm.experimental.convergence.anchor()
+  call void @f() [ "convergencectrl"(token %tok.l2) ]
+  br i1 %cond, label %l2, label %latch, !llvm.loop !1
+
+latch:
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  call void @f() [ "convergencectrl"(token %tok.l2) ]
+  ret i32 0
+}
+
+; Inner loop is extended inside the outer loop. Outer loop is unrolled.
+
+define i32 @extended_inner_loop_2(i32 %n, i1 %cond) {
+; CHECK-LABEL: @extended_inner_loop_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L3:%.*]]
+; CHECK:       l3:
+; CHECK-NEXT:    br label [[L2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2]]) ]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[L2]], label [[LATCH:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       latch:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2]]) ]
+; CHECK-NEXT:    br label [[L2_1:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.1:
+; CHECK-NEXT:    [[TOK_L2_1:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_1]], label [[LATCH_1:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       latch.1:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ]
+; CHECK-NEXT:    br label [[L2_2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.2:
+; CHECK-NEXT:    [[TOK_L2_2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_2]], label [[LATCH_2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       latch.2:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ]
+; CHECK-NEXT:    br label [[L2_3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.3:
+; CHECK-NEXT:    [[TOK_L2_3:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_3]], label [[LATCH_3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       latch.3:
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  %tok.loop = call token @llvm.experimental.convergence.anchor()
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 4
+  br label %l2, !llvm.loop !1
+
+l2:
+  %tok.l2 = call token @llvm.experimental.convergence.anchor()
+  call void @f() [ "convergencectrl"(token %tok.l2) ]
+  br i1 %cond, label %l2, label %latch, !llvm.loop !1
+
+latch:
+  call void @f() [ "convergencectrl"(token %tok.l2) ]
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+; No extension. Both loops unrolled.
+
+define i32 @unroll_nest(i32 %n, i1 %cond) {
+; CHECK-LABEL: @unroll_nest(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L3:%.*]]
+; CHECK:       l3:
+; CHECK-NEXT:    br label [[L2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2]]) ]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[L2_1:%.*]], label [[LATCH:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.1:
+; CHECK-NEXT:    [[TOK_L2_1:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2]], label [[LATCH]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       latch:
+; CHECK-NEXT:    br label [[L2_12:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.12:
+; CHECK-NEXT:    [[TOK_L2_11:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_11]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_1_1:%.*]], label [[LATCH_1:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.1.1:
+; CHECK-NEXT:    [[TOK_L2_1_1:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1_1]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_12]], label [[LATCH_1]], !llvm.loop [[LOOP9]]
+; CHECK:       latch.1:
+; CHECK-NEXT:    br label [[L2_2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.2:
+; CHECK-NEXT:    [[TOK_L2_2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_1_2:%.*]], label [[LATCH_2:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.1.2:
+; CHECK-NEXT:    [[TOK_L2_1_2:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1_2]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_2]], label [[LATCH_2]], !llvm.loop [[LOOP9]]
+; CHECK:       latch.2:
+; CHECK-NEXT:    br label [[L2_3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.3:
+; CHECK-NEXT:    [[TOK_L2_3:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_1_3:%.*]], label [[LATCH_3:%.*]], !llvm.loop [[LOOP4]]
+; CHECK:       l2.1.3:
+; CHECK-NEXT:    [[TOK_L2_1_3:%.*]] = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    call void @f() [ "convergencectrl"(token [[TOK_L2_1_3]]) ]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2_3]], label [[LATCH_3]], !llvm.loop [[LOOP9]]
+; CHECK:       latch.3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  %tok.loop = call token @llvm.experimental.convergence.anchor()
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 4
+  br label %l2, !llvm.loop !1
+
+l2:
+  %tok.l2 = call token @llvm.experimental.convergence.anchor()
+  call void @f() [ "convergencectrl"(token %tok.l2) ]
+  br i1 %cond, label %l2, label %latch, !llvm.loop !1
+
+latch:
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+declare token @llvm.experimental.convergence.anchor()
+declare token @llvm.experimental.convergence.loop()
+
+!0 = !{!0, !{!"llvm.loop.unroll.count", i32 16}}
+!1 = !{!1, !{!"llvm.loop.unroll.count", i32 2}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 32dc2ec1d50a8b378a89451655c92e7df0568a7d..56616d4fca4c923110937ee071b10d9cd5d34a20 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -120,8 +120,6 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) {
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x i32>, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP9]] = zext <2 x i32> [[WIDE_LOAD4]] to <2 x i64>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[TMP8]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP9]], <2 x i32> <i32 1, i32 2>
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -185,8 +183,6 @@ define i64 @int_and_pointer_iv(ptr %start, i32 %N) {
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP5]] = zext <4 x i32> [[WIDE_LOAD3]] to <4 x i64>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
index b7463032cada8eef9fd7b5587896a0a50616be55..fae23b6907b077ba3fa161ffaf6097efb6a21819 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -776,7 +776,6 @@ define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP4]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP4]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw <16 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc <16 x i32> [[TMP6]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP1]]
@@ -860,7 +859,6 @@ define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[TMP6]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP6]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <16 x i32> [[TMP6]], <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP9:%.*]] = trunc <16 x i32> [[TMP8]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index 809d2e8f7ea1ad6daa4236415f74dfef41b98b8b..34d5db4a40ddfc60f2bee487816c1cc13569b232 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -54,7 +54,6 @@ define i32 @pr70988() {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
-; CHECK-NEXT:    [[TMP21:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT7]], true
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index bcf8096f1b73834fceaf8c59af96ece194cfd42d..52d62b3309501839546d6241533df388e38ca0ac 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -481,9 +481,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP80:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP81:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT12]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP82:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT13]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP83:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT14]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP84:%.*]] = extractelement <vscale x 8 x i1> [[TMP80]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
@@ -1799,9 +1796,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP98:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP99:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT16]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP100:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT17]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP101:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT18]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP102:%.*]] = extractelement <vscale x 8 x i1> [[TMP98]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP102]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
@@ -2198,9 +2192,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP98:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP99:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT16]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP100:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT17]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-ORDERED-TF-NEXT:    [[TMP101:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT18]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP102:%.*]] = extractelement <vscale x 8 x i1> [[TMP98]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP102]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 2acc1ddbffea4633f0f0fd7fa6ec0a1a071fd67a..52a4b1761877d7930aac391fff20048d71ef38f9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -96,9 +96,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[TMP72:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP73:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT11]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP74:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT12]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP75:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT13]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <vscale x 4 x i1> [[TMP72]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP76]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -249,9 +246,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[TMP94:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP95:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT14]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP96:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT15]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP97:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT16]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <vscale x 4 x i1> [[TMP94]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP98]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
index 2017797288c29746eafc1250ae28fcd2fcf51894..89a513515b205200551fbe8496cabcc52ec3d948 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll
@@ -291,6 +291,150 @@ for.end:
   ret void
 }
 
+declare float @sinf(float) nounwind readnone
+define void @sinf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @sinf_v4f32(
+; CHECK: call <4 x float> @_simd_sin_f4(
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @sinf(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @sin(double) nounwind readnone
+define void @sin_v2f64(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @sin_v2f64(
+; CHECK: call <2 x double> @_simd_sin_d2(
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @sin(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @tanf(float) nounwind readnone
+define void @tanf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tanf_v4f32(
+; CHECK: call <4 x float> @_simd_tan_f4(
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @tanf(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @tan(double) nounwind readnone
+define void @tan_v2f64(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tan_v2f64(
+; CHECK: call <2 x double> @_simd_tan_d2(
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @tan(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.tan.f32(float) nounwind readnone
+define void @tan_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tan_v4f32_intrinsic(
+; CHECK: call <4 x float> @_simd_tan_f4(<4 x float>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds float, ptr %y, i64 %iv
+  %lv = load float, ptr %gep.y, align 4
+  %call = tail call float @llvm.tan.f32(float %lv)
+  %gep.x = getelementptr inbounds float, ptr %x, i64 %iv
+  store float %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare double @llvm.tan.f64(double) nounwind readnone
+define void @tan_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) {
+; CHECK-LABEL: @tan_v2f64_intrinsic(
+; CHECK: call <2 x double> @_simd_tan_d2(<2 x double>
+; CHECK: ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %gep.y = getelementptr inbounds double, ptr %y, i64 %iv
+  %lv = load double, ptr %gep.y, align 4
+  %call = tail call double @llvm.tan.f64(double %lv)
+  %gep.x = getelementptr inbounds double, ptr %x, i64 %iv
+  store double %call, ptr %gep.x, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 declare float @cbrtf(float) nounwind readnone
 define void @cbrtf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) {
 ; CHECK-LABEL: @cbrtf_v4f32(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index 2a552077a42b61773dcf9eae6dd8b75bfc728904..f0ae0093b2641dca467860382dc6ac7feca05136 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|exp|log|sin|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|trunc)" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|exp|log|sin|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|trunc)" --version 2
 
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE
@@ -1472,6 +1472,79 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr %out.ptr) {
   ret void
 }
 
+declare double @llvm.tan.f64(double)
+declare float @llvm.tan.f32(float)
+
+define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @tan_f64
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @tan_f64
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @tan_f64
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @tan_f64
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+  %in = load double, ptr %in.gep, align 8
+  %call = tail call double @llvm.tan.f64(double %in)
+  %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+  store double %call, ptr %out.gep, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
+; SLEEF-NEON-LABEL: define void @tan_f32
+; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; SLEEF-SVE-LABEL: define void @tan_f32
+; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+; ARMPL-NEON-LABEL: define void @tan_f32
+; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+;
+; ARMPL-SVE-LABEL: define void @tan_f32
+; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv
+  %in = load float, ptr %in.gep, align 8
+  %call = tail call float @llvm.tan.f32(float %in)
+  %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv
+  store float %call, ptr %out.gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
 declare double @llvm.trunc.f64(double)
 declare float @llvm.trunc.f32(float)
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
index 8d2820a245d9525fb6ae1569b88608e83aaf9ed8..1627292732b6a81f15d9ad9ca2ff9d1d861e2f7e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll
@@ -15,6 +15,11 @@ declare float @cosf(float) #0
 declare double @llvm.cos.f64(double) #0
 declare float @llvm.cos.f32(float) #0
 
+declare double @tan(double) #0
+declare float @tanf(float) #0
+declare double @llvm.tan.f64(double) #0
+declare float @llvm.tan.f32(float) #0
+
 declare double @pow(double, double) #0
 declare float @powf(float, float) #0
 declare double @llvm.pow.f64(double, double) #0
@@ -264,6 +269,114 @@ for.end:
   ret void
 }
 
+define void @tan_f64(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @amd_vrd4_tan(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+; CHECK-AVX512-VF8-LABEL: @tan_f64(
+; CHECK-AVX512-VF8:    [[TMP5:%.*]] = call <8 x double> @amd_vrd8_tan(<8 x double> [[TMP4:%.*]])
+; CHECK-AVX512-VF8:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @tan(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @tan_f32(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @amd_vrs4_tanf(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+; CHECK-AVX512-VF16-LABEL: @tan_f32(
+; CHECK-AVX512-VF16:    [[TMP5:%.*]] = call <16 x float> @amd_vrs16_tanf(<16 x float> [[TMP4:%.*]])
+; CHECK-AVX512-VF16:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @tanf(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @tan_f64_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @amd_vrd4_tan(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+; CHECK-AVX512-VF8-LABEL: @tan_f64_intrinsic(
+; CHECK-AVX512-VF8:    [[TMP5:%.*]] = call <8 x double> @amd_vrd8_tan(<8 x double> [[TMP4:%.*]])
+; CHECK-AVX512-VF8:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.tan.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @tan_f32_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @amd_vrs4_tanf(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+; CHECK-AVX512-VF16-LABEL: @tan_f32_intrinsic(
+; CHECK-AVX512-VF16:    [[TMP5:%.*]] = call <16 x float> @amd_vrs16_tanf(<16 x float> [[TMP4:%.*]])
+; CHECK-AVX512-VF16:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.tan.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) {
 ; CHECK-LABEL: @pow_f64(
 ; CHECK:    [[TMP8:%.*]] = call <4 x double> @amd_vrd4_pow(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 64f3b1c5235de863cee5113d87dc206196d62ebb..9210aa31e5d117545143cb8aa7367f12bad29111 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -233,7 +233,7 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    [[ADD4]] = fadd fast float [[ADD]], [[T2]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 32
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[T0]]
-; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR]], label [[LOOPEXIT]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR]], label [[LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loopexit:
 ; CHECK-NEXT:    [[ADD4_LCSSA:%.*]] = phi float [ [[ADD4]], [[FOR]] ], [ [[TMP124]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_END]]
@@ -271,3 +271,212 @@ for.end:
   ret float %s.0.lcssa
 }
 
+define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
+; CHECK-LABEL: @multi_exit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[UMAX6:%.*]] = call i64 @llvm.umax.i64(i64 [[B:%.*]], i64 1)
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX6]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
+; CHECK-NEXT:    [[UMIN7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i64 [[UMIN7]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 30
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[B]], i64 1)
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze i64 [[TMP3]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP4]], i64 [[A]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[UMIN]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 1, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC_1:%.*]], i64 8
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC_2:%.*]], i64 8
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC_1]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[SRC_2]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP12]]
+; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i64, ptr [[SRC_1]], align 8, !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT10:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT9]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <2 x i64> poison, i64 [[TMP14]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT13]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <2 x i64> [[BROADCAST_SPLAT10]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <2 x i64> [[BROADCAST_SPLAT10]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <2 x i64> [[BROADCAST_SPLAT14]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <2 x i64> [[BROADCAST_SPLAT14]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <2 x i1> [[TMP17]], [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <2 x i1> [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[TMP21:%.*]] = zext <2 x i1> [[TMP19]] to <2 x i8>
+; CHECK-NEXT:    [[TMP22:%.*]] = zext <2 x i1> [[TMP20]] to <2 x i8>
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i8> [[TMP22]], i32 1
+; CHECK-NEXT:    store i8 [[TMP23]], ptr [[DST]], align 1, !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV_1_WIDE:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT_WIDE:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL8]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[EC_1:%.*]] = icmp ult i64 [[IV_1_WIDE]], [[A]]
+; CHECK-NEXT:    br i1 [[EC_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[L_1:%.*]] = load i64, ptr [[SRC_1]], align 8
+; CHECK-NEXT:    [[L_2:%.*]] = load i64, ptr [[SRC_2]], align 8
+; CHECK-NEXT:    [[CMP55_US:%.*]] = icmp eq i64 [[L_1]], 0
+; CHECK-NEXT:    [[CMP_I_US:%.*]] = icmp ne i64 [[L_2]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP_I_US]], [[CMP55_US]]
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[AND]] to i8
+; CHECK-NEXT:    store i8 [[EXT]], ptr [[DST]], align 1
+; CHECK-NEXT:    [[IV_1_NEXT]] = add i32 [[IV_1]], 1
+; CHECK-NEXT:    [[IV_1_NEXT_WIDE]] = zext i32 [[IV_1_NEXT]] to i64
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp ult i64 [[IV_1_NEXT_WIDE]], [[B]]
+; CHECK-NEXT:    br i1 [[EC_2]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1.wide = phi i64 [ 0, %entry ], [ %iv.1.next.wide, %loop.latch ]
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.latch ]
+  %ec.1 = icmp ult i64 %iv.1.wide, %A
+  br i1 %ec.1, label %loop.latch, label %exit
+
+loop.latch:
+  %l.1 = load i64, ptr %src.1, align 8
+  %l.2 = load i64, ptr %src.2, align 8
+  %cmp55.us = icmp eq i64 %l.1, 0
+  %cmp.i.us = icmp ne i64 %l.2, 0
+  %and = and i1 %cmp.i.us, %cmp55.us
+  %ext = zext i1 %and to i8
+  store i8 %ext, ptr %dst, align 1
+  %iv.1.next = add i32 %iv.1, 1
+  %iv.1.next.wide = zext i32 %iv.1.next to i64
+  %ec.2 = icmp ult i64 %iv.1.next.wide, %B
+  br i1 %ec.2, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define i1 @any_of_cost(ptr %start, ptr %end) #0 {
+; CHECK-LABEL: @any_of_cost(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START:%.*]] to i64
+; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[END1]], [[START2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 40
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[N_VEC]], 40
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 40
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 40
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 80
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 120
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP7]]
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 8
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 8
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x ptr> [[TMP16]], ptr [[TMP15]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP18]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <2 x ptr> [[TMP20]], ptr [[TMP19]], i32 1
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq <2 x ptr> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq <2 x ptr> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <2 x i1> [[TMP22]], <i1 true, i1 true>
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <2 x i1> [[TMP23]], <i1 true, i1 true>
+; CHECK-NEXT:    [[TMP26]] = or <2 x i1> [[VEC_PHI]], [[TMP24]]
+; CHECK-NEXT:    [[TMP27]] = or <2 x i1> [[VEC_PHI3]], [[TMP25]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <2 x i1> [[TMP27]], [[TMP26]]
+; CHECK-NEXT:    [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[BIN_RDX]])
+; CHECK-NEXT:    [[TMP30:%.*]] = freeze i1 [[TMP29]]
+; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP30]], i1 false, i1 false
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ false, [[ENTRY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[ANY_OF:%.*]] = phi i1 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ANY_OF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 8
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP13_NOT_NOT:%.*]] = icmp eq ptr [[L]], null
+; CHECK-NEXT:    [[ANY_OF_NEXT]] = select i1 [[CMP13_NOT_NOT]], i1 [[ANY_OF]], i1 false
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 40
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ANY_OF_NEXT_LCSSA:%.*]] = phi i1 [ [[ANY_OF_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    ret i1 [[ANY_OF_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %any.of = phi i1 [ false, %entry ], [ %any.of.next, %loop ]
+  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
+  %gep = getelementptr i8, ptr %ptr.iv, i64 8
+  %l = load ptr, ptr %gep, align 8
+  %cmp13.not.not = icmp eq ptr %l, null
+  %any.of.next = select i1 %cmp13.not.not, i1 %any.of, i1 false
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 40
+  %cmp.not = icmp eq ptr %ptr.iv, %end
+  br i1 %cmp.not, label %exit, label %loop
+
+exit:
+  ret i1 %any.of.next
+}
+
+attributes #0 = { "target-cpu"="penryn" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index 94575004e364b4dfafbafa6313b2810bde381c52..d58b4f5053df6579c28b318229ca4b325b907c74 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -234,8 +234,6 @@ define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <16 x i64> [[VEC_IND]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw <16 x i64> zeroinitializer, [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP1]] = sub nsw <16 x i64> zeroinitializer, [[STEP_ADD]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i64> [[VECTOR_RECUR]], <16 x i64> [[TMP0]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i64> [[TMP0]], <16 x i64> [[TMP1]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i64> [[TMP1]], i32 15
 ; CHECK-NEXT:    store i64 [[TMP4]], ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll b/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll
index 038852f55f455810cebceb71e8ae78aecb049c5c..67a2cf2b80e70c941a81390f6bdfcd6537418c92 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll
@@ -356,6 +356,117 @@ for.end:                                          ; preds = %for.body
 !132 = !{!"llvm.loop.vectorize.width", i32 8}
 !133 = !{!"llvm.loop.vectorize.enable", i1 true}
 
+define void @tan_f64(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f64(
+; CHECK-LABEL:    vector.body
+; CHECK:    [[TMP5:%.*]] = call <2 x double> @_ZGVbN2v_tan(<2 x double> [[TMP4:%.*]])
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @tan(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+!141 = distinct !{!141, !142, !143}
+!142 = !{!"llvm.loop.vectorize.width", i32 2}
+!143 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+define void @tan_f32(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f32(
+; CHECK-LABEL:    vector.body
+; CHECK:    [[TMP5:%.*]] = call <8 x float> @_ZGVdN8v_tanf(<8 x float> [[TMP4:%.*]])
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @tanf(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21
+
+for.end:
+  ret void
+}
+
+!151 = distinct !{!151, !152, !153}
+!152 = !{!"llvm.loop.vectorize.width", i32 8}
+!153 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+define void @tan_f64_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f64_intrinsic(
+; CHECK-LABEL:    vector.body
+; CHECK:    [[TMP5:%.*]] = call <2 x double> @_ZGVbN2v_tan(<2 x double> [[TMP4:%.*]])
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.tan.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !31
+
+for.end:
+  ret void
+}
+
+!161 = distinct !{!161, !162, !163}
+!162 = !{!"llvm.loop.vectorize.width", i32 2}
+!163 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+define void @tan_f32_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f32_intrinsic(
+; CHECK-LABEL:    vector.body
+; CHECK:    [[TMP5:%.*]] = call <8 x float> @_ZGVdN8v_tanf(<8 x float> [[TMP4:%.*]])
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.tan.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !41
+
+for.end:
+  ret void
+}
+
+
+
+!171 = distinct !{!171, !172, !173}
+!172 = !{!"llvm.loop.vectorize.width", i32 8}
+!173 = !{!"llvm.loop.vectorize.enable", i1 true}
+
 attributes #0 = { nounwind readnone }
 
 declare double @sin(double) #0
@@ -366,6 +477,10 @@ declare double @cos(double) #0
 declare float @cosf(float) #0
 declare double @llvm.cos.f64(double) #0
 declare float @llvm.cos.f32(float) #0
+declare double @tan(double) #0
+declare float @tanf(float) #0
+declare double @llvm.tan.f64(double) #0
+declare float @llvm.tan.f32(float) #0
 declare float @expf(float) #0
 declare float @powf(float, float) #0
 declare float @llvm.exp.f32(float) #0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
index 829a91f18e5e20949cac6e17627416b58ba9a1c5..ec838986ea48b2bea74c8ad1cdeb642b1c51c4c1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
@@ -77,7 +77,6 @@ define void @test(ptr %p) {
 ; VEC-NEXT:    store i64 0, ptr [[TMP26]], align 8
 ; VEC-NEXT:    [[TMP27:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1>
 ; VEC-NEXT:    [[TMP28]] = zext <4 x i16> [[TMP27]] to <4 x i64>
-; VEC-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP28]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VEC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4>
 ; VEC-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
index 005557d7445caae9e7ff898e78f3f82688ffcf3e..2e78e3632feab65b2f12a7f80b6f6842071575c9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
@@ -230,6 +230,52 @@ for.end:
   ret void
 }
 
+define void @tan_f64_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_tan4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.tan.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv
+  store double %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @tan_f32_intrinsic(ptr nocapture %varray) {
+; CHECK-LABEL: @tan_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_tanf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.tan.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv
+  store float %call, ptr %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
 define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) {
 ; CHECK-LABEL: @pow_f64(
 ; CHECK:    [[TMP8:%.*]] = call <4 x double> @__svml_pow4(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
index 2e78a96a44b74788f99d0a30446e76904bc9f793..27038f3a24b664f985732b12daf677cbf08f99ec 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
@@ -406,6 +406,31 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
+;CHECK-LABEL: @tan_f32_intrinsic(
+;CHECK: vtanf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @llvm.tan.f32(float) nounwind readnone
+define void @tan_f32_intrinsic(i32 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, ptr %y, i64 %indvars.iv
+  %0 = load float, ptr %arrayidx, align 4
+  %call = tail call float @llvm.tan.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, ptr %x, i64 %indvars.iv
+  store float %call, ptr %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
 ;CHECK-LABEL: @asin_f32(
 ;CHECK: vasinf{{.*}}<4 x float>
 ;CHECK: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
index eeb270c4be5452e48f111f53218bc54b4dd447ba..3374e2e67e35e237dec0e889273f32db98221872 100644
--- a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
@@ -1,28 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -force-vector-width=2 -force-vector-interleave=2 -passes=loop-vectorize -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
-; CHECK-LABEL: @dead_instructions_01
 ;
 ; This test ensures that we don't generate trivially dead instructions prior to
 ; instruction simplification. We don't need to generate instructions
 ; corresponding to the original induction variable update or branch condition,
 ; since we rewrite the loop structure.
 ;
-; CHECK:     vector.body:
-; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
-; CHECK:       %[[I0:.+]] = add i64 %index, 0
-; CHECK:       %[[I2:.+]] = add i64 %index, 2
-; CHECK:       getelementptr inbounds i64, ptr %a, i64 %[[I0]]
-; CHECK:       getelementptr inbounds i64, ptr %a, i64 %[[I2]]
-; CHECK-NOT:   add nuw nsw i64 %[[I0]], 1
-; CHECK-NOT:   add nuw nsw i64 %[[I2]], 1
-; CHECK-NOT:   icmp slt i64 {{.*}}, %n
-; CHECK:       %index.next = add nuw i64 %index, 4
-; CHECK:       %[[CMP:.+]] = icmp eq i64 %index.next, %n.vec
-; CHECK:       br i1 %[[CMP]], label %middle.block, label %vector.body
-;
 define i64 @dead_instructions_01(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @dead_instructions_01(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP7]] = add <2 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <2 x i64> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R:%.*]] = phi i64 [ [[TMP2:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2]] = add i64 [[TMP1]], [[R]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ [[TMP2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
 entry:
   br label %for.body
 
@@ -42,16 +78,44 @@ for.end:
 }
 
 
-; CHECK-LABEL: @pr47390
 ;
 ; This test ensures that the primary induction is not considered dead when
 ; acting as the 'add' of another induction, and otherwise feeding only its own
 ; 'add' (recognized earlier as 'dead'), when the tail of the loop is folded by
 ; masking. Such masking uses the primary induction.
 ;
-; CHECK:     vector.body:
-;
 define void @pr47390(ptr %a) {
+; CHECK-LABEL: define void @pr47390(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 7, %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ 9, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[USE_PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PRIMARY]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SECONDARY:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[SECONDARY_ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PRIMARY_ADD]] = add i32 [[PRIMARY]], 1
+; CHECK-NEXT:    [[SECONDARY_ADD]] = add i32 [[SECONDARY]], 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[SECONDARY]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SECONDARY]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+;
 entry:
   br label %loop
 
@@ -69,3 +133,92 @@ loop:
   %cmp = icmp eq i32 %secondary, 5
   br i1 %cmp, label %exit, label %loop
 }
+
+; Test with a dead load and dead vector poiner.
+define void @dead_load_and_vector_pointer(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @dead_load_and_vector_pointer(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 516
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 516
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 8, !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP5]], align 8, !alias.scope [[META6]], !noalias [[META9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[WIDE_LOAD]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = add <2 x i32> [[WIDE_LOAD2]], <i32 1, i32 1>
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], ptr [[TMP4]], align 4, !alias.scope [[META6]], !noalias [[META9]]
+; CHECK-NEXT:    store <2 x i32> [[TMP7]], ptr [[TMP5]], align 4, !alias.scope [[META6]], !noalias [[META9]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PRIMARY_ADD]] = add i32 [[PRIMARY]], 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[PRIMARY]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LOAD]], 1
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[PRIMARY]]
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, ptr [[GEP_B]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PRIMARY]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
+;
+entry:
+  br label %loop
+
+exit:
+  ret void
+
+loop:
+  %primary = phi i32 [ 0, %entry ], [ %primary_add, %loop ]
+  %primary_add = add i32 %primary, 1
+  %gep = getelementptr inbounds i32, ptr %a, i32 %primary
+  %load = load i32, ptr %gep, align 8
+  %add = add i32 %load, 1
+  store i32 %add, ptr %gep
+  %gep.b = getelementptr inbounds i32, ptr %b, i32 %primary
+  %load2 = load i32, ptr %gep.b
+  %cmp = icmp eq i32 %primary, 128
+  br i1 %cmp, label %exit, label %loop
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
index 447f0b0bfee20f8b56ba1492c0715bd3ae917baa..8dd1e71e1f1d0ab1cda5728ccb2a96ce19dbf49c 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
@@ -560,7 +560,6 @@ define ptr @test_first_order_recurrences_and_pointer_induction1(ptr %ptr) {
 ; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %vector.ph ], [ [[TMP0:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP0]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x ptr> [[VECTOR_RECUR]], <4 x ptr> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds ptr, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    store <4 x ptr> [[TMP0]], ptr [[TMP4]], align 8
@@ -604,7 +603,6 @@ define ptr @test_first_order_recurrences_and_pointer_induction2(ptr %ptr) {
 ; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %vector.ph ], [ [[TMP0:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP0]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x ptr> [[VECTOR_RECUR]], <4 x ptr> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds ptr, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    store <4 x ptr> [[TMP0]], ptr [[TMP4]], align 8
@@ -653,7 +651,6 @@ define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x double> poison, double [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT3]], <4 x double> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[BROADCAST_SPLAT4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR2]], <4 x double> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
 ; CHECK-NEXT:    store double [[TMP6]], ptr [[P:%.*]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
index 937e01e69abdc998ea4af9428c01005f22423c28..7e15f038114fdde0ef7a676c8617960c3f594f08 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
@@ -220,22 +220,54 @@ exit:
 define void @test_pr54233_for_depend_on_each_other(ptr noalias %a, ptr noalias %b) {
 ; CHECK-LABEL: @test_pr54233_for_depend_on_each_other(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], <i32 10, i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[TMP4]] = xor <4 x i32> <i32 12, i32 12, i32 12, i32 12>, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP6]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP8:%.*]] = and <4 x i32> [[TMP7]], [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 3
+; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT2]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[FOR_2]], 10
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[FOR_2]], [[FOR_1]]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR4:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], [[SCALAR_PH]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SCALAR_RECUR4]], 10
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[SCALAR_RECUR4]], [[SCALAR_RECUR]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[SHL]], 255
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[XOR]], [[OR]]
-; CHECK-NEXT:    [[FOR_1_NEXT]] = xor i32 12, [[FOR_2]]
-; CHECK-NEXT:    [[FOR_2_NEXT]] = load i32, ptr [[B:%.*]], align 4
-; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = xor i32 12, [[SCALAR_RECUR4]]
+; CHECK-NEXT:    [[FOR_2_NEXT]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store i32 [[AND]], ptr [[A_GEP]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 1000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 6d3654d3b97e4b0cd242ae7bbf36f6823a76fe5c..665ac0e1fa4450ff69abdfb7f54087bb2f981cd2 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -911,8 +911,6 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4>
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 -4, i32 -4, i32 -4, i32 -4>
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -1006,7 +1004,6 @@ define i32 @PR27246() {
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4>
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -1108,10 +1105,6 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP20]], align 4
 ; UNROLL-NO-IC-NEXT:    [[TMP29:%.*]] = load i32, ptr [[TMP21]], align 4
 ; UNROLL-NO-IC-NEXT:    [[TMP30:%.*]] = load i32, ptr [[TMP22]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1
-; UNROLL-NO-IC-NEXT:    [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2
-; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3
 ; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = load i32, ptr [[TMP23]], align 4
 ; UNROLL-NO-IC-NEXT:    [[TMP36:%.*]] = load i32, ptr [[TMP24]], align 4
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = load i32, ptr [[TMP25]], align 4
@@ -1120,8 +1113,6 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1
 ; UNROLL-NO-IC-NEXT:    [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[VECTOR_RECUR_EXTRACT_FOR_PHI]], i32 2
 ; UNROLL-NO-IC-NEXT:    [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[VECTOR_RECUR_EXTRACT]], i32 3
-; UNROLL-NO-IC-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP34]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP34]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP45:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1227,7 +1218,6 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; SINK-AFTER-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1
 ; SINK-AFTER-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[VECTOR_RECUR_EXTRACT_FOR_PHI]], i32 2
 ; SINK-AFTER-NEXT:    [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[VECTOR_RECUR_EXTRACT]], i32 3
-; SINK-AFTER-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1276,7 +1266,6 @@ define i64 @constant_folded_previous_value() {
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1335,7 +1324,6 @@ define i64 @constant_folded_previous_value() {
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ <i64 1, i64 1, i64 1, i64 1>, [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; SINK-AFTER-NEXT:    br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1392,8 +1380,6 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
 ; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; UNROLL-NO-IC-NEXT:    [[TMP1]] = add <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]]
-; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 4, i32 4, i32 4, i32 4>
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
@@ -1463,7 +1449,6 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[TMP0]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SINK-AFTER-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
 ; SINK-AFTER-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
@@ -2571,8 +2556,6 @@ define void @sink_dead_inst(ptr %a) {
 ; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1>
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP5]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR2]], <4 x i32> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP2]], <i16 5, i16 5, i16 5, i16 5>
 ; UNROLL-NO-IC-NEXT:    [[TMP9]] = add <4 x i16> [[TMP3]], <i16 5, i16 5, i16 5, i16 5>
 ; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP8]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -2678,7 +2661,6 @@ define void @sink_dead_inst(ptr %a) {
 ; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1>
 ; SINK-AFTER-NEXT:    [[TMP2]] = zext <4 x i16> [[TMP1]] to <4 x i32>
-; SINK-AFTER-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR1]], <4 x i32> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP4]] = add <4 x i16> [[TMP1]], <i16 5, i16 5, i16 5, i16 5>
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP6:%.*]] = sub <4 x i16> [[TMP5]], <i16 10, i16 10, i16 10, i16 10>
@@ -3490,8 +3472,6 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = or <4 x i16> [[TMP3]], [[TMP3]]
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = zext <4 x i16> [[TMP4]] to <4 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP7]] = zext <4 x i16> [[TMP5]] to <4 x i32>
-; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[A_PTR:%.*]], i16 [[TMP0]]
 ; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[A_PTR]], i16 [[TMP1]]
 ; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0
@@ -3590,7 +3570,6 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1>
 ; SINK-AFTER-NEXT:    [[TMP2:%.*]] = or <4 x i16> [[TMP1]], [[TMP1]]
 ; SINK-AFTER-NEXT:    [[TMP3]] = zext <4 x i16> [[TMP2]] to <4 x i32>
-; SINK-AFTER-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[A_PTR:%.*]], i16 [[TMP0]]
 ; SINK-AFTER-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0
 ; SINK-AFTER-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP6]], align 4
@@ -3664,8 +3643,6 @@ define void @unused_recurrence(ptr %a) {
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[STEP_ADD]], <i16 1, i16 1, i16 1, i16 1>
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5>
 ; UNROLL-NO-IC-NEXT:    [[TMP3]] = add <4 x i16> [[TMP1]], <i16 5, i16 5, i16 5, i16 5>
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], <i16 4, i16 4, i16 4, i16 4>
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
@@ -3735,7 +3712,6 @@ define void @unused_recurrence(ptr %a) {
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], <i16 1, i16 1, i16 1, i16 1>
 ; SINK-AFTER-NEXT:    [[TMP1]] = add <4 x i16> [[TMP0]], <i16 5, i16 5, i16 5, i16 5>
-; SINK-AFTER-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], <i16 4, i16 4, i16 4, i16 4>
 ; SINK-AFTER-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1028
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index b959894b671e1505f5d6f06fdcb545ef202ff501..b3721056c4d7fd5e0ec6ed5a2b4885ef67a3b35d 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -171,8 +171,6 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-LABEL: @constant_folded_previous_value
 ; CHECK-VF4UF2: vector.body
 ; CHECK-VF4UF2: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i64> [ %vector.recur.init, %vector.ph ], [ shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), %vector.body ]
-; CHECK-VF4UF2: %[[SPLICE1:.*]] = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> %vector.recur, <vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), i32 -1)
-; CHECK-VF4UF2: %[[SPLICE2:.*]] = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), i32 -1)
 ; CHECK-VF4UF2: br i1 {{.*}}, label %middle.block, label %vector.body
 entry:
   br label %scalar.body
diff --git a/llvm/test/Transforms/PGOProfile/memprof.ll b/llvm/test/Transforms/PGOProfile/memprof.ll
index 13f370a4071e8aab6907a949294b594bd684b1f6..4a87f4f9d74494bf863afeb1c41cb2ac9d22f520 100644
--- a/llvm/test/Transforms/PGOProfile/memprof.ll
+++ b/llvm/test/Transforms/PGOProfile/memprof.ll
@@ -5,6 +5,8 @@
 ; REQUIRES: zlib
 ;; Avoid failures on big-endian systems that can't read the profile properly
 ; REQUIRES: x86_64-linux
+;; -stats requires asserts
+; REQUIRES: asserts
 
 ;; TODO: Use text profile inputs once that is available for memprof.
 ;; # To update the Inputs below, run Inputs/update_memprof_inputs.sh.
@@ -25,7 +27,7 @@
 ; ALL-NOT: no profile data available for function
 
 ;; Using a memprof-only profile for memprof-use should only give memprof metadata
-; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL,MEMPROFONLY
+; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-print-match-info -stats 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL,MEMPROFONLY,MEMPROFMATCHINFO,MEMPROFSTATS
 ; There should not be any PGO metadata
 ; MEMPROFONLY-NOT: !prof
 
@@ -61,6 +63,15 @@
 ;; give both memprof and pgo metadata.
 ; RUN: opt < %s -passes='pgo-instr-use,memprof-use<profile-filename=%t.pgomemprofdata>' -pgo-test-profile-file=%t.pgomemprofdata -pgo-warn-missing-function -S 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL,PGO
 
+; MEMPROFMATCHINFO: MemProf notcold context with id 1093248920606587996 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf notcold context with id 5725971306423925017 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf notcold context with id 6792096022461663180 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf cold context with id 8525406123785421946 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf cold context with id 11714230664165068698 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf cold context with id 15737101490731057601 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf cold context with id 16342802530253093571 has total profiled size 10 is matched
+; MEMPROFMATCHINFO: MemProf cold context with id 18254812774972004394 has total profiled size 10 is matched
+
 ; ModuleID = 'memprof.cc'
 source_filename = "memprof.cc"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -346,6 +357,14 @@ for.end:                                          ; preds = %for.cond
 ; MEMPROFNOCOLINFO: ![[C10]] = !{i64 -4535090212904553409}
 ; MEMPROFNOCOLINFO: ![[C11]] = !{i64 3577763375057267810}
 
+; MEMPROFSTATS:  8 memprof - Number of alloc contexts in memory profile.
+; MEMPROFSTATS: 10 memprof - Number of callsites in memory profile.
+; MEMPROFSTATS:  6 memprof - Number of functions having valid memory profile.
+; MEMPROFSTATS:  8 memprof - Number of matched memory profile alloc contexts.
+; MEMPROFSTATS:  3 memprof - Number of matched memory profile allocs.
+; MEMPROFSTATS: 10 memprof - Number of matched memory profile callsites.
+
+
 ; Function Attrs: argmemonly nofree nounwind willreturn writeonly
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #3
 
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 5cbf50e06fbe88bed1809211988fb7747a59a5f5..db16413cdc94af92dc7ff4e3f566dac5a0078ad3 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -182,11 +182,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK:       vector.body.1:
 ; CHECK-NEXT:    [[INDEX_1:%.*]] = phi i64 [ 0, [[VECTOR_PH_1]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY_1]] ]
 ; CHECK-NEXT:    [[TMP33:%.*]] = add nuw nsw i64 [[INDEX_1]], 15
-; CHECK-NEXT:    [[TMP34:%.*]] = add i64 [[INDEX_1]], 16
+; CHECK-NEXT:    [[TMP34:%.*]] = add nuw nsw i64 [[INDEX_1]], 16
 ; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[TMP33]], i64 0
 ; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <2 x i64> [[TMP35]], i64 [[TMP34]], i64 1
-; CHECK-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX_1]], 17
-; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[INDEX_1]], 18
+; CHECK-NEXT:    [[TMP37:%.*]] = add nuw nsw i64 [[INDEX_1]], 17
+; CHECK-NEXT:    [[TMP38:%.*]] = add nuw nsw i64 [[INDEX_1]], 18
 ; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <2 x i64> poison, i64 [[TMP37]], i64 0
 ; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <2 x i64> [[TMP39]], i64 [[TMP38]], i64 1
 ; CHECK-NEXT:    [[TMP41:%.*]] = icmp ult <2 x i64> [[TMP36]], <i64 225, i64 225>
@@ -259,11 +259,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK:       vector.body.2:
 ; CHECK-NEXT:    [[INDEX_2:%.*]] = phi i64 [ 0, [[VECTOR_PH_2]] ], [ [[INDEX_NEXT_2:%.*]], [[VECTOR_BODY_2]] ]
 ; CHECK-NEXT:    [[TMP64:%.*]] = add nuw nsw i64 [[INDEX_2]], 30
-; CHECK-NEXT:    [[TMP65:%.*]] = add i64 [[INDEX_2]], 31
+; CHECK-NEXT:    [[TMP65:%.*]] = add nuw nsw i64 [[INDEX_2]], 31
 ; CHECK-NEXT:    [[TMP66:%.*]] = insertelement <2 x i64> poison, i64 [[TMP64]], i64 0
 ; CHECK-NEXT:    [[TMP67:%.*]] = insertelement <2 x i64> [[TMP66]], i64 [[TMP65]], i64 1
-; CHECK-NEXT:    [[TMP68:%.*]] = add i64 [[INDEX_2]], 32
-; CHECK-NEXT:    [[TMP69:%.*]] = add i64 [[INDEX_2]], 33
+; CHECK-NEXT:    [[TMP68:%.*]] = add nuw nsw i64 [[INDEX_2]], 32
+; CHECK-NEXT:    [[TMP69:%.*]] = add nuw nsw i64 [[INDEX_2]], 33
 ; CHECK-NEXT:    [[TMP70:%.*]] = insertelement <2 x i64> poison, i64 [[TMP68]], i64 0
 ; CHECK-NEXT:    [[TMP71:%.*]] = insertelement <2 x i64> [[TMP70]], i64 [[TMP69]], i64 1
 ; CHECK-NEXT:    [[TMP72:%.*]] = icmp ult <2 x i64> [[TMP67]], <i64 225, i64 225>
@@ -336,11 +336,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK:       vector.body.3:
 ; CHECK-NEXT:    [[INDEX_3:%.*]] = phi i64 [ 0, [[VECTOR_PH_3]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY_3]] ]
 ; CHECK-NEXT:    [[TMP95:%.*]] = add nuw nsw i64 [[INDEX_3]], 45
-; CHECK-NEXT:    [[TMP96:%.*]] = add i64 [[INDEX_3]], 46
+; CHECK-NEXT:    [[TMP96:%.*]] = add nuw nsw i64 [[INDEX_3]], 46
 ; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <2 x i64> poison, i64 [[TMP95]], i64 0
 ; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <2 x i64> [[TMP97]], i64 [[TMP96]], i64 1
-; CHECK-NEXT:    [[TMP99:%.*]] = add i64 [[INDEX_3]], 47
-; CHECK-NEXT:    [[TMP100:%.*]] = add i64 [[INDEX_3]], 48
+; CHECK-NEXT:    [[TMP99:%.*]] = add nuw nsw i64 [[INDEX_3]], 47
+; CHECK-NEXT:    [[TMP100:%.*]] = add nuw nsw i64 [[INDEX_3]], 48
 ; CHECK-NEXT:    [[TMP101:%.*]] = insertelement <2 x i64> poison, i64 [[TMP99]], i64 0
 ; CHECK-NEXT:    [[TMP102:%.*]] = insertelement <2 x i64> [[TMP101]], i64 [[TMP100]], i64 1
 ; CHECK-NEXT:    [[TMP103:%.*]] = icmp ult <2 x i64> [[TMP98]], <i64 225, i64 225>
diff --git a/llvm/test/Transforms/Reassociate/repeats.ll b/llvm/test/Transforms/Reassociate/repeats.ll
index ba25c4bfc643cd2fdaa2e4520225f31c1ffc41be..8600777877bb3c4e7b1bceb62fc1b125a36c2fca 100644
--- a/llvm/test/Transforms/Reassociate/repeats.ll
+++ b/llvm/test/Transforms/Reassociate/repeats.ll
@@ -60,7 +60,8 @@ define i3 @foo3x5(i3 %x) {
 ; CHECK-SAME: i3 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i3 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i3 [[TMP3]], [[X]]
-; CHECK-NEXT:    ret i3 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i3 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    ret i3 [[TMP5]]
 ;
   %tmp1 = mul i3 %x, %x
   %tmp2 = mul i3 %tmp1, %x
@@ -74,7 +75,8 @@ define i3 @foo3x5_nsw(i3 %x) {
 ; CHECK-LABEL: define i3 @foo3x5_nsw(
 ; CHECK-SAME: i3 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i3 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw i3 [[TMP3]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i3 [[TMP3]], [[X]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i3 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret i3 [[TMP4]]
 ;
   %tmp1 = mul i3 %x, %x
@@ -89,7 +91,8 @@ define i3 @foo3x6(i3 %x) {
 ; CHECK-LABEL: define i3 @foo3x6(
 ; CHECK-SAME: i3 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i3 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i3 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i3 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i3 [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    ret i3 [[TMP2]]
 ;
   %tmp1 = mul i3 %x, %x
@@ -106,7 +109,9 @@ define i3 @foo3x7(i3 %x) {
 ; CHECK-SAME: i3 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i3 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i3 [[TMP5]], [[X]]
-; CHECK-NEXT:    ret i3 [[TMP6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i3 [[TMP6]], [[X]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i3 [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    ret i3 [[TMP7]]
 ;
   %tmp1 = mul i3 %x, %x
   %tmp2 = mul i3 %tmp1, %x
@@ -123,7 +128,8 @@ define i4 @foo4x8(i4 %x) {
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP1]], [[TMP1]]
-; CHECK-NEXT:    ret i4 [[TMP4]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP4]], [[TMP4]]
+; CHECK-NEXT:    ret i4 [[TMP3]]
 ;
   %tmp1 = mul i4 %x, %x
   %tmp2 = mul i4 %tmp1, %x
@@ -140,8 +146,9 @@ define i4 @foo4x9(i4 %x) {
 ; CHECK-LABEL: define i4 @foo4x9(
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i4 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP2]], [[X]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i4 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[TMP8]]
 ;
   %tmp1 = mul i4 %x, %x
@@ -160,7 +167,8 @@ define i4 @foo4x10(i4 %x) {
 ; CHECK-LABEL: define i4 @foo4x10(
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP4]], [[X]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP2]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[TMP3]]
 ;
@@ -181,7 +189,8 @@ define i4 @foo4x11(i4 %x) {
 ; CHECK-LABEL: define i4 @foo4x11(
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP4]], [[X]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP2]], [[X]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul i4 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[TMP10]]
@@ -204,7 +213,9 @@ define i4 @foo4x12(i4 %x) {
 ; CHECK-LABEL: define i4 @foo4x12(
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP4]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    ret i4 [[TMP2]]
 ;
   %tmp1 = mul i4 %x, %x
@@ -227,7 +238,9 @@ define i4 @foo4x13(i4 %x) {
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i4 [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i4 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP3]], [[X]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i4 [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    ret i4 [[TMP12]]
 ;
   %tmp1 = mul i4 %x, %x
@@ -252,7 +265,9 @@ define i4 @foo4x14(i4 %x) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i4 [[TMP1]], [[X]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul i4 [[TMP6]], [[TMP6]]
-; CHECK-NEXT:    ret i4 [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP7]], [[X]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i4 [[TMP4]], [[TMP4]]
+; CHECK-NEXT:    ret i4 [[TMP5]]
 ;
   %tmp1 = mul i4 %x, %x
   %tmp2 = mul i4 %tmp1, %x
@@ -276,8 +291,10 @@ define i4 @foo4x15(i4 %x) {
 ; CHECK-SAME: i4 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i4 [[X]], [[X]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i4 [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i4 [[TMP6]], [[X]]
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i4 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i4 [[TMP6]], [[TMP6]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i4 [[TMP3]], [[X]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i4 [[TMP4]], [[X]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i4 [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    ret i4 [[TMP14]]
 ;
   %tmp1 = mul i4 %x, %x
diff --git a/llvm/test/Transforms/Reassociate/shifttest.ll b/llvm/test/Transforms/Reassociate/shifttest.ll
index 102166e16c056683e392f8e2c88c9e2c14144c5c..84ac5396a3a2b297426a6b8d7b8f82c9c1c6489e 100644
--- a/llvm/test/Transforms/Reassociate/shifttest.ll
+++ b/llvm/test/Transforms/Reassociate/shifttest.ll
@@ -1,12 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=reassociate,instcombine -S | FileCheck %s
+
 ; With shl->mul reassociation, we can see that this is (shl A, 9) * A
+define i32 @test(i32 %A) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = shl i32 [[A]], 9
+; CHECK-NEXT:    [[Z:%.*]] = mul i32 [[Y]], [[A]]
+; CHECK-NEXT:    ret i32 [[Z]]
 ;
-; RUN: opt < %s -passes=reassociate,instcombine -S |\
-; RUN:    grep "shl .*, 9"
-
-define i32 @test(i32 %A, i32 %B) {
-	%X = shl i32 %A, 5		; <i32> [#uses=1]
-	%Y = shl i32 %A, 4		; <i32> [#uses=1]
-	%Z = mul i32 %Y, %X		; <i32> [#uses=1]
-	ret i32 %Z
+  %X = shl i32 %A, 5		; <i32> [#uses=1]
+  %Y = shl i32 %A, 4		; <i32> [#uses=1]
+  %Z = mul i32 %Y, %X		; <i32> [#uses=1]
+  ret i32 %Z
 }
 
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
index 97b0ed600ad106e5a2e36db0d38fd56a42757ce2..6d4429b37a52bd6871dcdcef45ef61db2102215e 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
@@ -64,8 +64,8 @@ attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "fra
 
 
 ; PROBE: ![[CALL1]] = !DILocation(line: 4, column: 3, scope: ![[CALL1BLOCK:[0-9]+]])
-; PROBE: ![[CALL1BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 186646559)
+; PROBE: ![[CALL1BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 455147551)
 ; PROBE: ![[CALL2]] = !DILocation(line: 4, column: 9, scope: ![[CALL2BLOCK:[0-9]+]])
-; PROBE: ![[CALL2BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 186646567)
+; PROBE: ![[CALL2BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 455344167)
 ; PROBE: ![[INST]] = !DILocation(line: 4, column: 15, scope: ![[INSTBLOCK:[0-9]+]])
 ; PROBE: ![[INSTBLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 4)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll
index 3d278a9ba27fd1e2a243e8aef12085f8fa7faba0..0bde361018f7d2d00795a6761f54368629f8d119 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll
@@ -42,13 +42,13 @@ define dso_local i32 @entry() !dbg !14 {
 ; CHECK-IL: ![[#SCOPE2:]] = distinct !DISubprogram(name: "foo"
 ; CHECK-IL: ![[#DL1]] = !DILocation(line: 3, column: 1,  scope: ![[#SCOPE1]], inlinedAt: ![[#INL1:]])
 ; CHECK-IL: ![[#INL1]] = distinct !DILocation(line: 7, column: 3, scope: ![[#BL1:]])
-;; A discriminator of 186646551 which is 0xb200017 in hexdecimal, stands for a direct call probe
+;; A discriminator of 455082007 which is 0x1b200017 in hexdecimal, stands for a direct call probe
 ;; with an index of 2 and a scale of 100%.
-; CHECK-IL: ![[#BL1]] = !DILexicalBlockFile(scope: ![[#SCOPE2]], file: !1, discriminator: 186646551)
+; CHECK-IL: ![[#BL1]] = !DILexicalBlockFile(scope: ![[#SCOPE2]], file: !1, discriminator: 455082007)
 ; CHECK-IL: ![[#SCOPE3:]] = distinct !DISubprogram(name: "entry"
 ; CHECK-IL: ![[#DL2]] = !DILocation(line: 7, column: 3,  scope: ![[#SCOPE2]], inlinedAt: ![[#INL2:]])
 ; CHECK-IL: ![[#INL2]] = distinct !DILocation(line: 11, column: 3, scope: ![[#BL2:]])
-; CHECK-IL: ![[#BL2]] = !DILexicalBlockFile(scope: ![[#SCOPE3]], file: !1, discriminator: 186646551)
+; CHECK-IL: ![[#BL2]] = !DILexicalBlockFile(scope: ![[#SCOPE3]], file: !1, discriminator: 455082007)
 ; CHECK-IL: ![[#DL3]] = !DILocation(line: 3, column: 1,  scope: ![[#SCOPE1]], inlinedAt: ![[#INL3:]])
 ; CHECK-IL: ![[#INL3]] = distinct !DILocation(line: 7, column: 3,  scope: ![[#BL1]], inlinedAt: ![[#INL2]])
 
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
index 13cfd820ae82c5f055d6040658ec0dd4f261d8d1..62d3d8255a1757550b2911341237613daa370ac1 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
@@ -84,13 +84,13 @@ entry:
 ; CHECK-IL: ![[#REALLINE]] = !DILocation(line: 2, scope: ![[#DISC0:]])
 ; CHECK-IL: ![[#DISC0]] = !DILexicalBlockFile(scope: ![[#FOO]], file: ![[#]], discriminator: 0)
 ; CHECK-IL: ![[#PROBE0]] = !DILocation(line: 2, column: 20, scope: ![[#SCOPE0:]])
-;; A discriminator of 67108887 which is 0x7200017 in hexdecimal, stands for a direct call probe
+;; A discriminator of 387973143 which is 0x17200017 in hexdecimal, stands for a direct call probe
 ;; with an index of 2.
-; CHECK-IL: ![[#SCOPE0]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 119537687)
+; CHECK-IL: ![[#SCOPE0]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 387973143)
 ; CHECK-IL: ![[#PROBE1]] = !DILocation(line: 0, scope: ![[#SCOPE1:]])
-;; A discriminator of 186646559 which is 0xb20001f in hexdecimal, stands for a direct call probe
+;; A discriminator of 455082015 which is 0x1b20001f in hexdecimal, stands for a direct call probe
 ;; with an index of 3.
-; CHECK-IL: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 186646559)
+; CHECK-IL: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 455082015)
 
 ; Check the generation of .pseudo_probe_desc section
 ; CHECK-ASM: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo,comdat
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
index 2cd9abf0e11e941e66437335b4c7984ef02a2f09..df3bc61d02520238228f87f2ab5b1291c902b9a7 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
@@ -106,8 +106,6 @@ if.end:
 ;YAML-NEXT:    - Line:            '1'
 ;YAML-NEXT:    - String:          ':'
 ;YAML-NEXT:    - Column:          '11'
-;YAML-NEXT:    - String:          .
-;YAML-NEXT:    - Disc:            '2'
 ;YAML-NEXT:    - String:          ';'
 ;YAML-NEXT:  ...
 ;YAML:  --- !Analysis
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll
index 7258ffca1278f34d5558f88f16f72247c3cdbe32..b52f93763d4926a8f734752c3d20a51ea3f57595 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll
@@ -40,14 +40,14 @@ attributes #0 = {"use-sample-profile"}
 
 ; CHECK: ![[PD1]] = !{!"branch_weights", i32 8, i32 7}
 ; CHECK: ![[#PROBE1]] = !DILocation(line: 0, scope: ![[#SCOPE1:]])
-;; A discriminator of 119537695 which is 0x720001f in hexdecimal, stands for an indirect call probe
+;; A discriminator of 387973151 which is 0x1720001f in hexdecimal, stands for an indirect call probe
 ;; with an index of 3.
-; CHECK: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 119537695)
+; CHECK: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 387973151)
 ; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 7, i64 9191153033785521275, i64 5, i64 -1069303473483922844, i64 2}
-;; A discriminator of 119537711 which is 0x720002f in hexdecimal, stands for an indirect call probe
+;; A discriminator of 387973167 which is 0x1720002f in hexdecimal, stands for an indirect call probe
 ;; with an index of 5.
 ; CHECK: ![[#PROBE2]] = !DILocation(line: 0, scope: ![[#SCOPE2:]])
-; CHECK: ![[#SCOPE2]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 119537711)
+; CHECK: ![[#SCOPE2]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 387973167)
 ; CHECK: ![[PROF2]] = !{!"VP", i32 0, i64 6, i64 -1069303473483922844, i64 4, i64 9191153033785521275, i64 2}
 
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-verify.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-verify.ll
index b622cfbd6634ef6b475bcf30e6764c7055c3e0d8..dccd37e9de99d958dda57d7cc4df2dac6cd623d1 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-verify.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-verify.ll
@@ -54,10 +54,10 @@ bb24:
   ret void
 }
 
-;; A discriminator of 186646583 which is 0xb200037 in hexdecimal, stands for a direct call probe
+;; A discriminator of 455082031 which is 0x1b200037 in hexdecimal, stands for a direct call probe
 ;; with an index of 6 and a scale of -1%.
 ; CHECK: ![[#PROBE6]] = !DILocation(line: 2, column: 20, scope: ![[#SCOPE:]])
-; CHECK: ![[#SCOPE]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 186646575)
+; CHECK: ![[#SCOPE]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 455082031)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
index 463a85cfd583ddef44a640cc0ec624682d424ad4..6def8f4eeb0891972a6dd5cb6bf8a55d26648a81 100644
--- a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=static    < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
 ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=pic       < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
 ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=ropi      < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
index 3ded78d85a610c065c7e30ac907002a135e4cbc9..2ac94afd9591056d3bb057d9de2db9bf3fcbbafd 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
@@ -34,10 +34,6 @@ define i32 @switch_of_powers(i32 %x) {
 ; RV64ZBB-LABEL: @switch_of_powers(
 ; RV64ZBB-NEXT:  entry:
 ; RV64ZBB-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
-; RV64ZBB-NEXT:    [[SWITCH_MASKINDEX:%.*]] = trunc i32 [[TMP0]] to i8
-; RV64ZBB-NEXT:    [[SWITCH_SHIFTED:%.*]] = lshr i8 121, [[SWITCH_MASKINDEX]]
-; RV64ZBB-NEXT:    [[SWITCH_LOBIT:%.*]] = trunc i8 [[SWITCH_SHIFTED]] to i1
-; RV64ZBB-NEXT:    call void @llvm.assume(i1 [[SWITCH_LOBIT]])
 ; RV64ZBB-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers, i32 0, i32 [[TMP0]]
 ; RV64ZBB-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; RV64ZBB-NEXT:    ret i32 [[SWITCH_LOAD]]
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll
index 6858746e87248bf770e1fd36033ebc74795f610f..18a3c9a6f7545a02a975f7840f5b4618b393a348 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll
@@ -1,26 +1,25 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mattr=+f | FileCheck %s
 ; RUN: opt < %s -passes='simplifycfg<no-keep-loops;switch-to-lookup>' -S -mattr=+f | FileCheck %s
 target datalayout = "e-m:e-p:32:32-i64:64-n32-S128"
 target triple = "riscv32-unknown-elf"
 
-; The table for @f
-; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
-
-; The char table for char
-; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
-
-; The float table for @h
-; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
-
-; The table for @foostring
-; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 4
-
 ; A simple int-to-int selection switch.
 ; It is dense enough to be replaced by table lookup.
 ; The result is directly by a ret from an otherwise empty bb,
 ; so we return early, directly from the lookup bb.
 
+;.
+; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1
+; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1
+; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1
+; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1
+; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1
+; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
+; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
+; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
+; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 4
+;.
 define i32 @f(i32 %c) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  entry:
@@ -189,3 +188,6 @@ return:
   ret ptr %retval.0
 
 }
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { "target-features"="+f" }
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
index f680cdbe6e7c56257c37dfbcfada5cce829cd8fc..6c9450bb8998edce8251a11561cc142e828e6669 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
@@ -1,26 +1,25 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mattr=+f | FileCheck %s
 ; RUN: opt < %s -passes='simplifycfg<no-keep-loops;switch-to-lookup>' -S -mattr=+f | FileCheck %s
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "riscv64-unknown-elf"
 
-; The table for @f
-; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
-
-; The char table for char
-; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
-
-; The float table for @h
-; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
-
-; The table for @foostring
-; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8
-
 ; A simple int-to-int selection switch.
 ; It is dense enough to be replaced by table lookup.
 ; The result is directly by a ret from an otherwise empty bb,
 ; so we return early, directly from the lookup bb.
 
+;.
+; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1
+; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1
+; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1
+; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1
+; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1
+; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
+; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
+; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
+; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8
+;.
 define i32 @f(i32 %c) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  entry:
@@ -189,3 +188,6 @@ return:
   ret ptr %retval.0
 
 }
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { "target-features"="+f" }
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index 9d6502072c16bcb48a25e49821582192edd72952..845c5008e3837b127a174581fcd7eb63d7fb5572 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -1,38 +1,45 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 ; RUN: opt < %s -passes='simplifycfg<no-keep-loops;switch-to-lookup>' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-; The table for @f
-; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
+; A simple int-to-int selection switch.
+; It is dense enough to be replaced by table lookup.
+; The result is directly by a ret from an otherwise empty bb,
+; so we return early, directly from the lookup bb.
 
-; The char table for char
+;.
+; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1
+; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1
+; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1
+; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1
+; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1
+; CHECK: @tls_a = thread_local global i32 0
+; CHECK: @tls_b = thread_local global i32 0
+; CHECK: @tls_c = thread_local global i32 0
+; CHECK: @tls_d = thread_local global i32 0
+; CHECK: @dllimport_a = external dllimport global [3 x i32]
+; CHECK: @dllimport_b = external dllimport global [3 x i32]
+; CHECK: @dllimport_c = external dllimport global [3 x i32]
+; CHECK: @dllimport_d = external dllimport global [3 x i32]
+; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
 ; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
-
-; The float table for @h
 ; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
-
-; The table for @foostring
 ; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8
-
-; The table for @earlyreturncrash
 ; CHECK: @switch.table.earlyreturncrash = private unnamed_addr constant [4 x i32] [i32 42, i32 9, i32 88, i32 5], align 4
-
-; The table for @large
-; CHECK: @switch.table.large = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9,
-
-; The table for @cprop
+; CHECK: @switch.table.earlyreturncrash.1 = private unnamed_addr constant [4 x i32] [i32 3, i32 4, i32 1, i32 5], align 4
+; CHECK: @switch.table.large = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9, i32 16, i32 25, i32 36, i32 49, i32 64, i32 81, i32 100, i32 121, i32 144, i32 169, i32 196, i32 225, i32 256, i32 289, i32 342, i32 361, i32 400, i32 441, i32 484, i32 529, i32 576, i32 625, i32 676, i32 729, i32 784, i32 841, i32 900, i32 961, i32 1024, i32 1089, i32 1156, i32 1260, i32 1296, i32 1369, i32 1444, i32 1521, i32 1600, i32 1681, i32 1764, i32 1849, i32 1980, i32 2025, i32 2116, i32 2209, i32 2304, i32 2401, i32 2500, i32 2601, i32 2704, i32 2809, i32 2970, i32 3025, i32 3136, i32 3249, i32 3364, i32 3481, i32 3600, i32 3721, i32 3844, i32 3969, i32 4096, i32 4225, i32 4356, i32 4489, i32 4624, i32 4761, i32 4900, i32 5112, i32 5184, i32 5329, i32 5476, i32 5625, i32 5776, i32 5929, i32 6084, i32 6241, i32 6400, i32 6561, i32 6724, i32 6889, i32 7056, i32 7225, i32 7396, i32 7569, i32 7744, i32 7921, i32 8100, i32 8281, i32 8464, i32 8649, i32 8836, i32 9025, i32 9216, i32 9409, i32 9604, i32 9801, i32 10000, i32 10201, i32 10404, i32 10609, i32 10816, i32 11025, i32 11236, i32 11449, i32 11664, i32 11881, i32 12100, i32 12321, i32 12544, i32 12769, i32 12996, i32 13225, i32 13456, i32 13689, i32 13924, i32 14161, i32 14400, i32 14641, i32 14884, i32 15129, i32 15376, i32 15625, i32 15876, i32 16129, i32 16384, i32 16641, i32 16900, i32 17161, i32 17424, i32 17689, i32 17956, i32 18225, i32 18496, i32 18769, i32 19044, i32 19321, i32 19600, i32 19881, i32 20164, i32 20449, i32 20736, i32 21025, i32 21316, i32 21609, i32 21904, i32 22201, i32 22500, i32 22801, i32 23104, i32 23409, i32 23716, i32 24025, i32 24336, i32 24649, i32 24964, i32 25281, i32 25600, i32 25921, i32 26244, i32 26569, i32 26896, i32 27225, i32 27556, i32 27889, i32 28224, i32 28561, i32 28900, i32 29241, i32 29584, i32 29929, i32 30276, i32 30625, i32 30976, i32 31329, i32 31684, i32 32041, i32 32400, i32 32761, i32 33124, i32 33489, i32 33856, i32 34225, i32 34596, i32 34969, i32 35344, i32 35721, i32 36100, i32 36481, i32 36864, i32 37249, i32 37636, i32 38025, i32 38416, i32 38809, i32 39204, i32 39601], align 4
 ; CHECK: @switch.table.cprop = private unnamed_addr constant [7 x i32] [i32 5, i32 42, i32 126, i32 -452, i32 128, i32 6, i32 7], align 4
-
-; The table for @unreachable_case
 ; CHECK: @switch.table.unreachable_case = private unnamed_addr constant [9 x i32] [i32 0, i32 0, i32 0, i32 2, i32 -1, i32 1, i32 1, i32 1, i32 1], align 4
-
-; A simple int-to-int selection switch.
-; It is dense enough to be replaced by table lookup.
-; The result is directly by a ret from an otherwise empty bb,
-; so we return early, directly from the lookup bb.
-
+; CHECK: @switch.table.unreachable_default = private unnamed_addr constant [4 x i32] [i32 42, i32 52, i32 1, i32 2], align 4
+; CHECK: @switch.table.nodefaultnoholes = private unnamed_addr constant [4 x i32] [i32 55, i32 123, i32 0, i32 -1], align 4
+; CHECK: @switch.table.nodefaultwithholes = private unnamed_addr constant [6 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 55, i32 -1], align 4
+; CHECK: @switch.table.threecases = private unnamed_addr constant [3 x i32] [i32 10, i32 7, i32 5], align 4
+; CHECK: @switch.table.covered_switch_with_bit_tests = private unnamed_addr constant [8 x i32] [i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 1, i32 1], align 4
+; CHECK: @switch.table.signed_overflow1 = private unnamed_addr constant [4 x i32] [i32 3333, i32 4444, i32 1111, i32 2222], align 4
+; CHECK: @switch.table.signed_overflow2 = private unnamed_addr constant [4 x i32] [i32 3333, i32 4444, i32 2222, i32 2222], align 4
+;.
 define i32 @f(i32 %c) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  entry:
@@ -308,10 +315,10 @@ define i32 @overflow(i32 %type) {
 ; CHECK-LABEL: @overflow(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    switch i32 [[TYPE:%.*]], label [[IF_END:%.*]] [
-; CHECK-NEXT:    i32 3, label [[SW_BB3:%.*]]
-; CHECK-NEXT:    i32 -2147483645, label [[SW_BB3]]
-; CHECK-NEXT:    i32 1, label [[SW_BB1:%.*]]
-; CHECK-NEXT:    i32 2, label [[SW_BB2:%.*]]
+; CHECK-NEXT:      i32 3, label [[SW_BB3:%.*]]
+; CHECK-NEXT:      i32 -2147483645, label [[SW_BB3]]
+; CHECK-NEXT:      i32 1, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[IF_END]]
@@ -931,11 +938,11 @@ define i96 @illegaltype(i32 %c) {
 ; CHECK-LABEL: @illegaltype(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 42, label [[RETURN:%.*]]
-; CHECK-NEXT:    i32 43, label [[SW_BB1:%.*]]
-; CHECK-NEXT:    i32 44, label [[SW_BB2:%.*]]
-; CHECK-NEXT:    i32 45, label [[SW_BB3:%.*]]
-; CHECK-NEXT:    i32 46, label [[SW_BB4:%.*]]
+; CHECK-NEXT:      i32 42, label [[RETURN:%.*]]
+; CHECK-NEXT:      i32 43, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 44, label [[SW_BB2:%.*]]
+; CHECK-NEXT:      i32 45, label [[SW_BB3:%.*]]
+; CHECK-NEXT:      i32 46, label [[SW_BB4:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[RETURN]]
@@ -1050,9 +1057,9 @@ define i32 @threecasesholes(i32 %c) {
 ; CHECK-LABEL: @threecasesholes(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 0, label [[RETURN:%.*]]
-; CHECK-NEXT:    i32 1, label [[SW_BB1:%.*]]
-; CHECK-NEXT:    i32 3, label [[SW_BB2:%.*]]
+; CHECK-NEXT:      i32 0, label [[RETURN:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 3, label [[SW_BB2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[RETURN]]
@@ -1142,9 +1149,9 @@ define ptr @tls(i32 %x) {
 ; CHECK-LABEL: @tls(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    switch i32 [[X:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 0, label [[RETURN:%.*]]
-; CHECK-NEXT:    i32 1, label [[SW_BB1:%.*]]
-; CHECK-NEXT:    i32 2, label [[SW_BB2:%.*]]
+; CHECK-NEXT:      i32 0, label [[RETURN:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[RETURN]]
@@ -1182,9 +1189,9 @@ define ptr @dllimport(i32 %x) {
 ; CHECK-LABEL: @dllimport(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    switch i32 [[X:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 0, label [[RETURN:%.*]]
-; CHECK-NEXT:    i32 1, label [[SW_BB1:%.*]]
-; CHECK-NEXT:    i32 2, label [[SW_BB2:%.*]]
+; CHECK-NEXT:      i32 0, label [[RETURN:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[RETURN]]
@@ -1476,11 +1483,11 @@ define void @pr20210(i8 %x, i1 %y) {
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[SW:%.*]], label [[INTERMEDIATE:%.*]]
 ; CHECK:       sw:
 ; CHECK-NEXT:    switch i8 [[X:%.*]], label [[END:%.*]] [
-; CHECK-NEXT:    i8 7, label [[INTERMEDIATE]]
-; CHECK-NEXT:    i8 3, label [[INTERMEDIATE]]
-; CHECK-NEXT:    i8 2, label [[INTERMEDIATE]]
-; CHECK-NEXT:    i8 1, label [[INTERMEDIATE]]
-; CHECK-NEXT:    i8 0, label [[INTERMEDIATE]]
+; CHECK-NEXT:      i8 7, label [[INTERMEDIATE]]
+; CHECK-NEXT:      i8 3, label [[INTERMEDIATE]]
+; CHECK-NEXT:      i8 2, label [[INTERMEDIATE]]
+; CHECK-NEXT:      i8 1, label [[INTERMEDIATE]]
+; CHECK-NEXT:      i8 0, label [[INTERMEDIATE]]
 ; CHECK-NEXT:    ]
 ; CHECK:       intermediate:
 ; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[X]] to i32
@@ -1612,15 +1619,15 @@ define void @wineh_test(i64 %val) personality ptr @__CxxFrameHandler3 {
 ; CHECK-LABEL: @wineh_test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    invoke void @throw(i1 false)
-; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[CLEANUP1:%.*]]
+; CHECK-NEXT:            to label [[UNREACHABLE:%.*]] unwind label [[CLEANUP1:%.*]]
 ; CHECK:       unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cleanup1:
 ; CHECK-NEXT:    [[CLEANUPPAD1:%.*]] = cleanuppad within none []
 ; CHECK-NEXT:    switch i64 [[VAL:%.*]], label [[CLEANUPDONE2:%.*]] [
-; CHECK-NEXT:    i64 0, label [[CLEANUPDONE1:%.*]]
-; CHECK-NEXT:    i64 1, label [[CLEANUPDONE1]]
-; CHECK-NEXT:    i64 6, label [[CLEANUPDONE1]]
+; CHECK-NEXT:      i64 0, label [[CLEANUPDONE1:%.*]]
+; CHECK-NEXT:      i64 1, label [[CLEANUPDONE1]]
+; CHECK-NEXT:      i64 6, label [[CLEANUPDONE1]]
 ; CHECK-NEXT:    ]
 ; CHECK:       cleanupdone1:
 ; CHECK-NEXT:    cleanupret from [[CLEANUPPAD1]] unwind label [[CLEANUP2:%.*]]
@@ -1732,12 +1739,53 @@ define i32 @signed_overflow2(i8 %n) {
 ; CHECK-LABEL: @signed_overflow2(
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
-; CHECK-NEXT:    switch i2 [[TRUNC]], label [[BB1:%.*]] [
-; CHECK-NEXT:    i2 1, label [[BB6:%.*]]
-; CHECK-NEXT:    i2 -2, label [[BB4:%.*]]
-; CHECK-NEXT:    i2 -1, label [[BB5:%.*]]
+; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TRUNC]], -2
+; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow2, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+;
+start:
+  %trunc = trunc i8 %n to i2
+  switch i2 %trunc, label %bb1 [
+  i2 1, label %bb3
+  i2 -2, label %bb4
+  i2 -1, label %bb5
+  ]
+
+bb1:                                              ; preds = %start
+  unreachable
+
+bb3:                                              ; preds = %start
+  br label %bb6
+
+bb4:                                              ; preds = %start
+  br label %bb6
+
+bb5:                                              ; preds = %start
+  br label %bb6
+
+bb6:                                              ; preds = %start, %bb3, %bb4, %bb5
+  %.sroa.0.0 = phi i32 [ 4444, %bb5 ], [ 3333, %bb4 ], [ 2222, %bb3 ]
+  ret i32 %.sroa.0.0
+}
+
+; This is the same as @signed_overflow2 except that the default case calls @exit(), so it
+; isn't treated as unreachable
+define i32 @signed_overflow3(i8 %n) {
+; CHECK-LABEL: @signed_overflow3(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
+; CHECK-NEXT:    switch i2 [[TRUNC]], label [[START_UNREACHABLEDEFAULT:%.*]] [
+; CHECK-NEXT:      i2 1, label [[BB6:%.*]]
+; CHECK-NEXT:      i2 -2, label [[BB4:%.*]]
+; CHECK-NEXT:      i2 -1, label [[BB5:%.*]]
+; CHECK-NEXT:      i2 0, label [[BB1:%.*]]
 ; CHECK-NEXT:    ]
+; CHECK:       start.unreachabledefault:
+; CHECK-NEXT:    unreachable
 ; CHECK:       bb1:
+; CHECK-NEXT:    call void @exit(i32 1)
 ; CHECK-NEXT:    unreachable
 ; CHECK:       bb4:
 ; CHECK-NEXT:    br label [[BB6]]
@@ -1756,6 +1804,7 @@ start:
   ]
 
 bb1:                                              ; preds = %start
+  call void @exit(i32 1)
   unreachable
 
 bb3:                                              ; preds = %start
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table_big.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table_big.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7988e3057a2c23925223d9944a46229be690b8ed
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table_big.ll
@@ -0,0 +1,542 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-pc-linux-gnu"
+
+; A dense switch with a reachable default case should be optimized into a lookup table with a bounds check
+;.
+; CHECK: @switch.table.reachable_default_dense_0to31 = private unnamed_addr constant [32 x i32] [i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1], align 4
+; CHECK: @switch.table.unreachable_default_dense_0to31 = private unnamed_addr constant [32 x i32] [i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1], align 4
+; CHECK: @switch.table.reachable_default_holes_0to31 = private unnamed_addr constant [32 x i32] [i32 0, i32 7, i32 6, i32 0, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 0, i32 2, i32 1, i32 0, i32 7, i32 0, i32 5, i32 4, i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 5, i32 0, i32 3, i32 2, i32 1], align 4
+; CHECK: @switch.table.unreachable_default_holes_0to31 = private unnamed_addr constant [32 x i32] [i32 0, i32 7, i32 6, i32 0, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 0, i32 2, i32 1, i32 0, i32 7, i32 0, i32 5, i32 4, i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 5, i32 0, i32 3, i32 2, i32 1], align 4
+; CHECK: @switch.table.reachable_default_dense_0to32 = private unnamed_addr constant [33 x i32] [i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0], align 4
+; CHECK: @switch.table.unreachable_default_dense_0to32 = private unnamed_addr constant [33 x i32] [i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0], align 4
+; CHECK: @switch.table.unreachable_default_holes_0to32 = private unnamed_addr constant [33 x i32] [i32 0, i32 7, i32 6, i32 0, i32 4, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 0, i32 2, i32 1, i32 0, i32 7, i32 0, i32 5, i32 4, i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 5, i32 0, i32 3, i32 2, i32 1, i32 0], align 4
+;.
+define i32 @reachable_default_dense_0to31(i32 %x, i32 %y) {
+; CHECK-LABEL: @reachable_default_dense_0to31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 32
+; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
+; CHECK:       switch.lookup:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [32 x i32], ptr @switch.table.reachable_default_dense_0to31, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SWITCH_LOAD]], [[SWITCH_LOOKUP]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 3, label %bb5
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 8, label %bb0
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 13, label %bb3
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 18, label %bb6
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 23, label %bb1
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 28, label %bb4
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  ]
+
+sw.default: br label %return
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ %y, %sw.default ], [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A dense switch with an unreachable default case should be optimized into a lookup table without bounds checks
+define i32 @unreachable_default_dense_0to31(i32 %x, i32 %y) {
+; CHECK-LABEL: @unreachable_default_dense_0to31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [32 x i32], ptr @switch.table.unreachable_default_dense_0to31, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 3, label %bb5
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 8, label %bb0
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 13, label %bb3
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 18, label %bb6
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 23, label %bb1
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 28, label %bb4
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  ]
+
+sw.default: unreachable
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A sparse switch with a reachable default case should be optimized into a lookup table with a bounds check and a mask
+define i32 @reachable_default_holes_0to31(i32 %x, i32 %y) {
+; CHECK-LABEL: @reachable_default_holes_0to31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 32
+; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_HOLE_CHECK:%.*]], label [[RETURN:%.*]]
+; CHECK:       switch.hole_check:
+; CHECK-NEXT:    [[SWITCH_SHIFTED:%.*]] = lshr i32 -277094665, [[X]]
+; CHECK-NEXT:    [[SWITCH_LOBIT:%.*]] = trunc i32 [[SWITCH_SHIFTED]] to i1
+; CHECK-NEXT:    br i1 [[SWITCH_LOBIT]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN]]
+; CHECK:       switch.lookup:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [32 x i32], ptr @switch.table.reachable_default_holes_0to31, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SWITCH_LOAD]], [[SWITCH_LOOKUP]] ], [ [[Y:%.*]], [[SWITCH_HOLE_CHECK]] ], [ [[Y]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  ]
+
+sw.default: br label %return
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ %y, %sw.default ], [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A sparse switch with an unreachable default case should be optimized into a lookup table without bounds checks
+define i32 @unreachable_default_holes_0to31(i32 %x, i32 %y) {
+; CHECK-LABEL: @unreachable_default_holes_0to31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [32 x i32], ptr @switch.table.unreachable_default_holes_0to31, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  ]
+
+sw.default: unreachable
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A dense switch with a reachable default case should be optimized into a lookup table with a bounds check
+define i32 @reachable_default_dense_0to32(i32 %x, i32 %y) {
+; CHECK-LABEL: @reachable_default_dense_0to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 33
+; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
+; CHECK:       switch.lookup:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [33 x i32], ptr @switch.table.reachable_default_dense_0to32, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[SWITCH_LOAD]], [[SWITCH_LOOKUP]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 3, label %bb5
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 8, label %bb0
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 13, label %bb3
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 18, label %bb6
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 23, label %bb1
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 28, label %bb4
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  i32 32, label %bb0
+  ]
+
+sw.default: br label %return
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ %y, %sw.default ], [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A dense switch with an unreachable default case should be optimized into a lookup table without bounds checks
+define i32 @unreachable_default_dense_0to32(i32 %x, i32 %y) {
+; CHECK-LABEL: @unreachable_default_dense_0to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [33 x i32], ptr @switch.table.unreachable_default_dense_0to32, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 3, label %bb5
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 8, label %bb0
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 13, label %bb3
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 18, label %bb6
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 23, label %bb1
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 28, label %bb4
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  i32 32, label %bb0
+  ]
+
+sw.default: unreachable
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A sparse switch with a reachable default case which would be optimized into a lookup table with a bounds check and a mask, but doesn't because
+; it would require a 33-bit mask
+define i32 @reachable_default_holes_0to32(i32 %x, i32 %y) {
+; CHECK-LABEL: @reachable_default_holes_0to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[RETURN:%.*]] [
+; CHECK-NEXT:      i32 0, label [[BB0:%.*]]
+; CHECK-NEXT:      i32 1, label [[BB7:%.*]]
+; CHECK-NEXT:      i32 2, label [[BB6:%.*]]
+; CHECK-NEXT:      i32 4, label [[BB4:%.*]]
+; CHECK-NEXT:      i32 5, label [[BB3:%.*]]
+; CHECK-NEXT:      i32 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i32 7, label [[BB1:%.*]]
+; CHECK-NEXT:      i32 9, label [[BB7]]
+; CHECK-NEXT:      i32 10, label [[BB6]]
+; CHECK-NEXT:      i32 11, label [[BB5:%.*]]
+; CHECK-NEXT:      i32 12, label [[BB4]]
+; CHECK-NEXT:      i32 14, label [[BB2]]
+; CHECK-NEXT:      i32 15, label [[BB1]]
+; CHECK-NEXT:      i32 16, label [[BB0]]
+; CHECK-NEXT:      i32 17, label [[BB7]]
+; CHECK-NEXT:      i32 19, label [[BB5]]
+; CHECK-NEXT:      i32 20, label [[BB4]]
+; CHECK-NEXT:      i32 21, label [[BB3]]
+; CHECK-NEXT:      i32 22, label [[BB2]]
+; CHECK-NEXT:      i32 24, label [[BB0]]
+; CHECK-NEXT:      i32 25, label [[BB7]]
+; CHECK-NEXT:      i32 26, label [[BB6]]
+; CHECK-NEXT:      i32 27, label [[BB5]]
+; CHECK-NEXT:      i32 29, label [[BB3]]
+; CHECK-NEXT:      i32 30, label [[BB2]]
+; CHECK-NEXT:      i32 31, label [[BB1]]
+; CHECK-NEXT:      i32 32, label [[BB0]]
+; CHECK-NEXT:    ]
+; CHECK:       bb0:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb5:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[BB0]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ], [ 3, [[BB3]] ], [ 4, [[BB4]] ], [ 5, [[BB5]] ], [ 6, [[BB6]] ], [ 7, [[BB7]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  i32 32, label %bb0
+  ]
+
+sw.default: br label %return
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ %y, %sw.default ], [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
+
+; A sparse switch with an unreachable default case which can be optimized into a lookup table without bounds checks. Because the default case is
+; unreachable, the fact that a 33-bit mask would be required doesn't prevent lookup table optimization.
+define i32 @unreachable_default_holes_0to32(i32 %x, i32 %y) {
+; CHECK-LABEL: @unreachable_default_holes_0to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [33 x i32], ptr @switch.table.unreachable_default_holes_0to32, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+;
+entry:
+  switch i32 %x, label %sw.default [
+  i32 0, label %bb0
+  i32 1, label %bb7
+  i32 2, label %bb6
+  i32 4, label %bb4
+  i32 5, label %bb3
+  i32 6, label %bb2
+  i32 7, label %bb1
+  i32 9, label %bb7
+  i32 10, label %bb6
+  i32 11, label %bb5
+  i32 12, label %bb4
+  i32 14, label %bb2
+  i32 15, label %bb1
+  i32 16, label %bb0
+  i32 17, label %bb7
+  i32 19, label %bb5
+  i32 20, label %bb4
+  i32 21, label %bb3
+  i32 22, label %bb2
+  i32 24, label %bb0
+  i32 25, label %bb7
+  i32 26, label %bb6
+  i32 27, label %bb5
+  i32 29, label %bb3
+  i32 30, label %bb2
+  i32 31, label %bb1
+  i32 32, label %bb0
+  ]
+
+sw.default: unreachable
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+bb4: br label %return
+bb5: br label %return
+bb6: br label %return
+bb7: br label %return
+
+return:
+  %res = phi i32 [ 0, %bb0 ], [ 1, %bb1 ], [ 2, %bb2 ], [ 3, %bb3 ], [ 4, %bb4 ], [ 5, %bb5 ], [ 6, %bb6 ], [ 7, %bb7 ]
+  ret i32 %res
+
+}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity-concat.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity-concat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d72532963d12ace84968b69e2ab57870d07a3199
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity-concat.ll
@@ -0,0 +1,446 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s
+
+target triple = "aarch64"
+
+define <8 x i8> @concat_small(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: @concat_small(
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[A:%.*]], <4 x i8> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %r = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i8> %r
+}
+
+define <8 x i32> @concat_big(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @concat_big(
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %r
+}
+
+define <8 x i8> @concatinit_addmul_small(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
+; CHECK-LABEL: @concatinit_addmul_small(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <8 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = add <8 x i8> [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %cb = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ct = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %xb = mul <4 x i8> %ab, %bb
+  %xt = mul <4 x i8> %at, %bt
+  %yb = add <4 x i8> %xb, %cb
+  %yt = add <4 x i8> %xt, %ct
+  %r = shufflevector <4 x i8> %yb, <4 x i8> %yt, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i8> %r
+}
+
+define <8 x i32> @concatinit_addmul_big(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) {
+; CHECK-LABEL: @concatinit_addmul_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <8 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = add <8 x i32> [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %ab = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %at = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %bb = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bt = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %cb = shufflevector <8 x i32> %c, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ct = shufflevector <8 x i32> %c, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %xb = mul <4 x i32> %ab, %bb
+  %xt = mul <4 x i32> %at, %bt
+  %yb = add <4 x i32> %xb, %cb
+  %yt = add <4 x i32> %xt, %ct
+  %r = shufflevector <4 x i32> %yb, <4 x i32> %yt, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %r
+}
+
+define <8 x i8> @concata_addmul_small(<4 x i8> %a1, <4 x i8> %a2, <8 x i8> %b, <8 x i8> %c) {
+; CHECK-LABEL: @concata_addmul_small(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[A1:%.*]], <4 x i8> [[A2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i8> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = add <8 x i8> [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %cb = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ct = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %xb = mul <4 x i8> %a1, %bb
+  %xt = mul <4 x i8> %a2, %bt
+  %yb = add <4 x i8> %xb, %cb
+  %yt = add <4 x i8> %xt, %ct
+  %r = shufflevector <4 x i8> %yb, <4 x i8> %yt, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i8> %r
+}
+
+define <8 x i32> @concata_addmul_big(<4 x i32> %a1, <4 x i32> %a2, <8 x i32> %b, <8 x i32> %c) {
+; CHECK-LABEL: @concata_addmul_big(
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CB:%.*]] = shufflevector <8 x i32> [[C:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[CT:%.*]] = shufflevector <8 x i32> [[C]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[XB:%.*]] = mul <4 x i32> [[A1:%.*]], [[BB]]
+; CHECK-NEXT:    [[XT:%.*]] = mul <4 x i32> [[A2:%.*]], [[BT]]
+; CHECK-NEXT:    [[YB:%.*]] = add <4 x i32> [[XB]], [[CB]]
+; CHECK-NEXT:    [[YT:%.*]] = add <4 x i32> [[XT]], [[CT]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[YB]], <4 x i32> [[YT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %bb = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bt = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %cb = shufflevector <8 x i32> %c, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ct = shufflevector <8 x i32> %c, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %xb = mul <4 x i32> %a1, %bb
+  %xt = mul <4 x i32> %a2, %bt
+  %yb = add <4 x i32> %xb, %cb
+  %yt = add <4 x i32> %xt, %ct
+  %r = shufflevector <4 x i32> %yb, <4 x i32> %yt, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %r
+}
+
+define <16 x i32> @concata_addmul_bigger(<4 x i32> %a1a, <4 x i32> %a2a, <4 x i32> %a3a, <4 x i32> %a4a, <16 x i32> %b, <16 x i32> %c) {
+; CHECK-LABEL: @concata_addmul_bigger(
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A1A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A2:%.*]] = shufflevector <4 x i32> [[A2A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <4 x i32> [[A3A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A4:%.*]] = shufflevector <4 x i32> [[A4A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <16 x i32> [[B:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[B2:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[B3:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[B4:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[C1:%.*]] = shufflevector <16 x i32> [[C:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[C2:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[C3:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[C4:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[X1:%.*]] = mul <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[X2:%.*]] = mul <4 x i32> [[A2]], [[B2]]
+; CHECK-NEXT:    [[X3:%.*]] = mul <4 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[X4:%.*]] = mul <4 x i32> [[A4]], [[B4]]
+; CHECK-NEXT:    [[Y1:%.*]] = add <4 x i32> [[X1]], [[C1]]
+; CHECK-NEXT:    [[Y2:%.*]] = add <4 x i32> [[X2]], [[C2]]
+; CHECK-NEXT:    [[Y3:%.*]] = add <4 x i32> [[X3]], [[C3]]
+; CHECK-NEXT:    [[Y4:%.*]] = add <4 x i32> [[X4]], [[C4]]
+; CHECK-NEXT:    [[CC1:%.*]] = shufflevector <4 x i32> [[Y1]], <4 x i32> [[Y2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CC2:%.*]] = shufflevector <4 x i32> [[Y3]], <4 x i32> [[Y4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[CC1]], <8 x i32> [[CC2]], <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %a1 = shufflevector <4 x i32> %a1a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a2 = shufflevector <4 x i32> %a2a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a3 = shufflevector <4 x i32> %a3a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a4 = shufflevector <4 x i32> %a4a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %b2 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %b3 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %b4 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %c1 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %c2 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %c3 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %c4 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %x1 = mul <4 x i32> %a1, %b1
+  %x2 = mul <4 x i32> %a2, %b2
+  %x3 = mul <4 x i32> %a3, %b3
+  %x4 = mul <4 x i32> %a4, %b4
+  %y1 = add <4 x i32> %x1, %c1
+  %y2 = add <4 x i32> %x2, %c2
+  %y3 = add <4 x i32> %x3, %c3
+  %y4 = add <4 x i32> %x4, %c4
+  %cc1 = shufflevector <4 x i32> %y1, <4 x i32> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cc2 = shufflevector <4 x i32> %y3, <4 x i32> %y4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x i32> %cc1, <8 x i32> %cc2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @concata_addmul_bigger_undef(<4 x i32> %a1a, <4 x i32> %a2a, <4 x i32> %a3a, <4 x i32> %a4a, <16 x i32> %b, <16 x i32> %c) {
+; CHECK-LABEL: @concata_addmul_bigger_undef(
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A1A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A2:%.*]] = shufflevector <4 x i32> [[A2A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <4 x i32> [[A3A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A4:%.*]] = shufflevector <4 x i32> [[A4A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <16 x i32> [[B:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[B2:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[B3:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[B4:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[C1:%.*]] = shufflevector <16 x i32> [[C:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[C2:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[C3:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[C4:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[X1:%.*]] = mul <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[X2:%.*]] = mul <4 x i32> [[A2]], [[B2]]
+; CHECK-NEXT:    [[X3:%.*]] = mul <4 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[X4:%.*]] = mul <4 x i32> [[A4]], [[B4]]
+; CHECK-NEXT:    [[Y1:%.*]] = add <4 x i32> [[X1]], [[C1]]
+; CHECK-NEXT:    [[Y2:%.*]] = add <4 x i32> [[X2]], [[C2]]
+; CHECK-NEXT:    [[Y3:%.*]] = add <4 x i32> [[X3]], [[C3]]
+; CHECK-NEXT:    [[Y4:%.*]] = add <4 x i32> [[X4]], [[C4]]
+; CHECK-NEXT:    [[CC1:%.*]] = shufflevector <4 x i32> [[Y1]], <4 x i32> [[Y2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 7>
+; CHECK-NEXT:    [[CC2:%.*]] = shufflevector <4 x i32> [[Y3]], <4 x i32> [[Y4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[CC1]], <8 x i32> [[CC2]], <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %a1 = shufflevector <4 x i32> %a1a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a2 = shufflevector <4 x i32> %a2a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a3 = shufflevector <4 x i32> %a3a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a4 = shufflevector <4 x i32> %a4a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %b2 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %b3 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %b4 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %c1 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %c2 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %c3 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %c4 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %x1 = mul <4 x i32> %a1, %b1
+  %x2 = mul <4 x i32> %a2, %b2
+  %x3 = mul <4 x i32> %a3, %b3
+  %x4 = mul <4 x i32> %a4, %b4
+  %y1 = add <4 x i32> %x1, %c1
+  %y2 = add <4 x i32> %x2, %c2
+  %y3 = add <4 x i32> %x3, %c3
+  %y4 = add <4 x i32> %x4, %c4
+  %cc1 = shufflevector <4 x i32> %y1, <4 x i32> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 7>
+  %cc2 = shufflevector <4 x i32> %y3, <4 x i32> %y4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x i32> %cc1, <8 x i32> %cc2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @splat_concat(<4 x i32> %a1a, <4 x i32> %a2a, <4 x i32> %a3a, <4 x i32> %a4a, <16 x i32> %b, <16 x i32> %c) {
+; CHECK-LABEL: @splat_concat(
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A1A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A2:%.*]] = shufflevector <4 x i32> [[A2A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <4 x i32> [[A3A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A4:%.*]] = shufflevector <4 x i32> [[A4A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <16 x i32> [[B:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[B2:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[B3:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[B4:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[C1:%.*]] = shufflevector <16 x i32> [[C:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[C2:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[C3:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[C4:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[SPLATA:%.*]] = shufflevector <4 x i32> [[A4A]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[X1:%.*]] = mul <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[X2:%.*]] = mul <4 x i32> [[A2]], [[B2]]
+; CHECK-NEXT:    [[X3:%.*]] = mul <4 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[X4:%.*]] = mul <4 x i32> [[A4]], [[B4]]
+; CHECK-NEXT:    [[Y1:%.*]] = add <4 x i32> [[X1]], [[C1]]
+; CHECK-NEXT:    [[Y2:%.*]] = add <4 x i32> [[X2]], [[C2]]
+; CHECK-NEXT:    [[Y3:%.*]] = add <4 x i32> [[X3]], [[C3]]
+; CHECK-NEXT:    [[Y4:%.*]] = add <4 x i32> [[X4]], [[C4]]
+; CHECK-NEXT:    [[Z1:%.*]] = xor <4 x i32> [[Y1]], [[SPLATA]]
+; CHECK-NEXT:    [[Z2:%.*]] = xor <4 x i32> [[Y2]], [[SPLATA]]
+; CHECK-NEXT:    [[Z3:%.*]] = xor <4 x i32> [[Y3]], [[SPLATA]]
+; CHECK-NEXT:    [[Z4:%.*]] = xor <4 x i32> [[Y4]], [[SPLATA]]
+; CHECK-NEXT:    [[CC1:%.*]] = shufflevector <4 x i32> [[Z1]], <4 x i32> [[Z2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CC2:%.*]] = shufflevector <4 x i32> [[Z3]], <4 x i32> [[Z4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[CC1]], <8 x i32> [[CC2]], <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %a1 = shufflevector <4 x i32> %a1a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a2 = shufflevector <4 x i32> %a2a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a3 = shufflevector <4 x i32> %a3a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a4 = shufflevector <4 x i32> %a4a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %b2 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %b3 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %b4 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %c1 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %c2 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %c3 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %c4 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %splata = shufflevector <4 x i32> %a4a, <4 x i32> poison, <4 x i32> zeroinitializer
+  %x1 = mul <4 x i32> %a1, %b1
+  %x2 = mul <4 x i32> %a2, %b2
+  %x3 = mul <4 x i32> %a3, %b3
+  %x4 = mul <4 x i32> %a4, %b4
+  %y1 = add <4 x i32> %x1, %c1
+  %y2 = add <4 x i32> %x2, %c2
+  %y3 = add <4 x i32> %x3, %c3
+  %y4 = add <4 x i32> %x4, %c4
+  %z1 = xor <4 x i32> %y1, %splata
+  %z2 = xor <4 x i32> %y2, %splata
+  %z3 = xor <4 x i32> %y3, %splata
+  %z4 = xor <4 x i32> %y4, %splata
+  %cc1 = shufflevector <4 x i32> %z1, <4 x i32> %z2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cc2 = shufflevector <4 x i32> %z3, <4 x i32> %z4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x i32> %cc1, <8 x i32> %cc2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @two_concats(<4 x i32> %a1a, <4 x i32> %a2a, <4 x i32> %a3a, <4 x i32> %a4a, <16 x i32> %b, <16 x i32> %c) {
+; CHECK-LABEL: @two_concats(
+; CHECK-NEXT:    [[A1:%.*]] = shufflevector <4 x i32> [[A1A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A2:%.*]] = shufflevector <4 x i32> [[A2A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <4 x i32> [[A3A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[A4:%.*]] = shufflevector <4 x i32> [[A4A:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[B1:%.*]] = shufflevector <16 x i32> [[B:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[B2:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[B3:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[B4:%.*]] = shufflevector <16 x i32> [[B]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[C1:%.*]] = shufflevector <16 x i32> [[C:%.*]], <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[C2:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[C3:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[C4:%.*]] = shufflevector <16 x i32> [[C]], <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[X1:%.*]] = mul <4 x i32> [[A1]], [[B1]]
+; CHECK-NEXT:    [[X2:%.*]] = mul <4 x i32> [[A2]], [[B2]]
+; CHECK-NEXT:    [[X3:%.*]] = mul <4 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[X4:%.*]] = mul <4 x i32> [[A4]], [[B4]]
+; CHECK-NEXT:    [[Y1:%.*]] = add <4 x i32> [[X1]], [[C1]]
+; CHECK-NEXT:    [[Y2:%.*]] = add <4 x i32> [[X2]], [[C2]]
+; CHECK-NEXT:    [[Y3:%.*]] = add <4 x i32> [[X3]], [[C3]]
+; CHECK-NEXT:    [[Y4:%.*]] = add <4 x i32> [[X4]], [[C4]]
+; CHECK-NEXT:    [[Z1:%.*]] = xor <4 x i32> [[Y1]], [[A1]]
+; CHECK-NEXT:    [[Z2:%.*]] = xor <4 x i32> [[Y2]], [[A1]]
+; CHECK-NEXT:    [[Z3:%.*]] = xor <4 x i32> [[Y3]], [[A1]]
+; CHECK-NEXT:    [[Z4:%.*]] = xor <4 x i32> [[Y4]], [[A1]]
+; CHECK-NEXT:    [[CC1:%.*]] = shufflevector <4 x i32> [[Z1]], <4 x i32> [[Z2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CC2:%.*]] = shufflevector <4 x i32> [[Z3]], <4 x i32> [[Z4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[CC1]], <8 x i32> [[CC2]], <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %a1 = shufflevector <4 x i32> %a1a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a2 = shufflevector <4 x i32> %a2a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a3 = shufflevector <4 x i32> %a3a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %a4 = shufflevector <4 x i32> %a4a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %b1 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %b2 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %b3 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %b4 = shufflevector <16 x i32> %b, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %c1 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 15, i32 14, i32 13, i32 12>
+  %c2 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 11, i32 10, i32 9, i32 8>
+  %c3 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %c4 = shufflevector <16 x i32> %c, <16 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %x1 = mul <4 x i32> %a1, %b1
+  %x2 = mul <4 x i32> %a2, %b2
+  %x3 = mul <4 x i32> %a3, %b3
+  %x4 = mul <4 x i32> %a4, %b4
+  %y1 = add <4 x i32> %x1, %c1
+  %y2 = add <4 x i32> %x2, %c2
+  %y3 = add <4 x i32> %x3, %c3
+  %y4 = add <4 x i32> %x4, %c4
+  %z1 = xor <4 x i32> %y1, %a1
+  %z2 = xor <4 x i32> %y2, %a1
+  %z3 = xor <4 x i32> %y3, %a1
+  %z4 = xor <4 x i32> %y4, %a1
+  %cc1 = shufflevector <4 x i32> %z1, <4 x i32> %z2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cc2 = shufflevector <4 x i32> %z3, <4 x i32> %z4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x i32> %cc1, <8 x i32> %cc2, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %r
+}
+
+
+define <16 x double> @konkat(<16 x double> %wide.vec, <16 x double> %wide.vec115, <2 x double> %l27, <2 x double> %l28, <2 x double> %l29, <2 x double> %l30) {
+; CHECK-LABEL: @konkat(
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[L27:%.*]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[L32:%.*]] = shufflevector <2 x double> [[L27]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT:    [[BROADCAST_SPLAT102:%.*]] = shufflevector <2 x double> [[L32]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLAT104:%.*]] = shufflevector <2 x double> [[L28:%.*]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[L33:%.*]] = shufflevector <2 x double> [[L28]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT:    [[BROADCAST_SPLAT106:%.*]] = shufflevector <2 x double> [[L33]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLAT108:%.*]] = shufflevector <2 x double> [[L29:%.*]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[L34:%.*]] = shufflevector <2 x double> [[L29]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT:    [[BROADCAST_SPLAT110:%.*]] = shufflevector <2 x double> [[L34]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLAT112:%.*]] = shufflevector <2 x double> [[L30:%.*]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[L35:%.*]] = shufflevector <2 x double> [[L30]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT:    [[BROADCAST_SPLAT114:%.*]] = shufflevector <2 x double> [[L35]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x double> [[WIDE_VEC:%.*]], <16 x double> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[STRIDED_VEC94:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC95:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC96:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[STRIDED_VEC97:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[STRIDED_VEC98:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[STRIDED_VEC99:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[STRIDED_VEC100:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC94]], [[BROADCAST_SPLAT102]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC95]], [[BROADCAST_SPLAT104]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC96]], [[BROADCAST_SPLAT106]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC97]], [[BROADCAST_SPLAT108]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC98]], [[BROADCAST_SPLAT110]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC99]], [[BROADCAST_SPLAT112]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul reassoc nsz contract <2 x double> [[STRIDED_VEC100]], [[BROADCAST_SPLAT114]]
+; CHECK-NEXT:    [[STRIDED_VEC116:%.*]] = shufflevector <16 x double> [[WIDE_VEC115:%.*]], <16 x double> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[STRIDED_VEC117:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC118:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC119:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[STRIDED_VEC120:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[STRIDED_VEC121:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[STRIDED_VEC122:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[STRIDED_VEC123:%.*]] = shufflevector <16 x double> [[WIDE_VEC115]], <16 x double> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC116]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC117]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC118]], [[TMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC119]], [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC120]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC121]], [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC122]], [[TMP7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fadd reassoc nsz contract <2 x double> [[STRIDED_VEC123]], [[TMP8]]
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> [[TMP18]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP21]], <8 x double> [[TMP22]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    ret <16 x double> [[INTERLEAVED_VEC]]
+;
+  %broadcast.splat = shufflevector <2 x double> %l27, <2 x double> poison, <2 x i32> zeroinitializer
+  %l32 = shufflevector <2 x double> %l27, <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+  %broadcast.splat102 = shufflevector <2 x double> %l32, <2 x double> poison, <2 x i32> zeroinitializer
+  %broadcast.splat104 = shufflevector <2 x double> %l28, <2 x double> poison, <2 x i32> zeroinitializer
+  %l33 = shufflevector <2 x double> %l28, <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+  %broadcast.splat106 = shufflevector <2 x double> %l33, <2 x double> poison, <2 x i32> zeroinitializer
+  %broadcast.splat108 = shufflevector <2 x double> %l29, <2 x double> poison, <2 x i32> zeroinitializer
+  %l34 = shufflevector <2 x double> %l29, <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+  %broadcast.splat110 = shufflevector <2 x double> %l34, <2 x double> poison, <2 x i32> zeroinitializer
+  %broadcast.splat112 = shufflevector <2 x double> %l30, <2 x double> poison, <2 x i32> zeroinitializer
+  %l35 = shufflevector <2 x double> %l30, <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+  %broadcast.splat114 = shufflevector <2 x double> %l35, <2 x double> poison, <2 x i32> zeroinitializer
+  %strided.vec = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 0, i32 8>
+  %strided.vec94 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 1, i32 9>
+  %strided.vec95 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 2, i32 10>
+  %strided.vec96 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 3, i32 11>
+  %strided.vec97 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 4, i32 12>
+  %strided.vec98 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 5, i32 13>
+  %strided.vec99 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 6, i32 14>
+  %strided.vec100 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 7, i32 15>
+  %37 = fmul reassoc nsz contract <2 x double> %strided.vec, %broadcast.splat
+  %38 = fmul reassoc nsz contract <2 x double> %strided.vec94, %broadcast.splat102
+  %39 = fmul reassoc nsz contract <2 x double> %strided.vec95, %broadcast.splat104
+  %40 = fmul reassoc nsz contract <2 x double> %strided.vec96, %broadcast.splat106
+  %41 = fmul reassoc nsz contract <2 x double> %strided.vec97, %broadcast.splat108
+  %42 = fmul reassoc nsz contract <2 x double> %strided.vec98, %broadcast.splat110
+  %43 = fmul reassoc nsz contract <2 x double> %strided.vec99, %broadcast.splat112
+  %44 = fmul reassoc nsz contract <2 x double> %strided.vec100, %broadcast.splat114
+  %strided.vec116 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 0, i32 8>
+  %strided.vec117 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 1, i32 9>
+  %strided.vec118 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 2, i32 10>
+  %strided.vec119 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 3, i32 11>
+  %strided.vec120 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 4, i32 12>
+  %strided.vec121 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 5, i32 13>
+  %strided.vec122 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 6, i32 14>
+  %strided.vec123 = shufflevector <16 x double> %wide.vec115, <16 x double> poison, <2 x i32> <i32 7, i32 15>
+  %46 = fadd reassoc nsz contract <2 x double> %strided.vec116, %37
+  %47 = fadd reassoc nsz contract <2 x double> %strided.vec117, %38
+  %48 = fadd reassoc nsz contract <2 x double> %strided.vec118, %39
+  %49 = fadd reassoc nsz contract <2 x double> %strided.vec119, %40
+  %50 = fadd reassoc nsz contract <2 x double> %strided.vec120, %41
+  %51 = fadd reassoc nsz contract <2 x double> %strided.vec121, %42
+  %52 = fadd reassoc nsz contract <2 x double> %strided.vec122, %43
+  %53 = fadd reassoc nsz contract <2 x double> %strided.vec123, %44
+  %54 = shufflevector <2 x double> %46, <2 x double> %47, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %55 = shufflevector <2 x double> %48, <2 x double> %49, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %56 = shufflevector <2 x double> %50, <2 x double> %51, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %57 = shufflevector <2 x double> %52, <2 x double> %53, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %58 = shufflevector <4 x double> %54, <4 x double> %55, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %59 = shufflevector <4 x double> %56, <4 x double> %57, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x double> %58, <8 x double> %59, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret <16 x double> %interleaved.vec
+}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 3e54099fc09701c0db6acf7e2083a94e5f3d20a6..c020aeb6483f5c2f6e20b69d2f1442748abb605f 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -938,15 +938,14 @@ define <4 x float> @fadd_mismatched_types(<4 x float> %x, <4 x float> %y) {
 }
 
 define void @maximal_legal_fpmath(ptr %addr1, ptr %addr2, ptr %result, float %val) {
-; CHECK-LABEL: define void @maximal_legal_fpmath(
-; CHECK-SAME: ptr [[ADDR1:%.*]], ptr [[ADDR2:%.*]], ptr [[RESULT:%.*]], float [[VAL:%.*]]) {
-; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[VAL]], i64 0
+; CHECK-LABEL: @maximal_legal_fpmath(
+; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[VAL:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[SPLATINSERT]], <4 x float> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC1:%.*]] = load <16 x float>, ptr [[ADDR1]], align 4
-; CHECK-NEXT:    [[VEC2:%.*]] = load <16 x float>, ptr [[ADDR2]], align 4
+; CHECK-NEXT:    [[VEC1:%.*]] = load <16 x float>, ptr [[ADDR1:%.*]], align 4
+; CHECK-NEXT:    [[VEC2:%.*]] = load <16 x float>, ptr [[ADDR2:%.*]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <16 x float> [[TMP1]], [[VEC2]]
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = fadd reassoc contract <16 x float> [[VEC1]], [[TMP2]]
-; CHECK-NEXT:    store <16 x float> [[INTERLEAVED_VEC]], ptr [[RESULT]], align 4
+; CHECK-NEXT:    store <16 x float> [[INTERLEAVED_VEC]], ptr [[RESULT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %splatinsert = insertelement <4 x float> poison, float %val, i64 0
diff --git a/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll b/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
index 7f7790cecb0eb83419b0d0d662fb6c895aa8d231..9ed62d3545f0764ab16fc78ac961fbd82338dc7a 100644
--- a/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
+++ b/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
@@ -1,19 +1,5 @@
 ; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
 
-declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1)
-define void @buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i1 %bool) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i1 %bool
-  ; CHECK-NEXT: %data0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 %bool, i1 false)
-  %data0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 %bool, i1 false)
-
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i1 %bool
-  ; CHECK-NEXT: %data1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 %bool)
-  %data1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 %bool)
-  ret void
-}
-
 declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32)
 define void @raw_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %arg) {
   ; CHECK: immarg operand has non-immediate parameter
@@ -665,12 +651,3 @@ define void @test_mfma_f32_32x32x1f32(float %arg0, float %arg1, <32 x i32> %arg2
 
   ret void
 }
-
-declare void @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1)
-define amdgpu_cs void @test_buffer_atomic_fadd(float %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %offset, i1 %slc) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i1 %slc
-  ; CHECK-NEXT: call void @llvm.amdgcn.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %offset, i1 %slc)
-  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %offset, i1 %slc)
-  ret void
-}
diff --git a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f75db964e2d6e97cf3de82a7ef2c7fa781f3f8a0
--- /dev/null
+++ b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir
@@ -0,0 +1,10 @@
+# REQUIRES: amdgpu-registered-target
+# RUN: llc -mtriple=amdgcn --passes='regallocfast<filter=sgpr>' --print-pipeline-passes --filetype=null %s | FileCheck %s --check-prefix=PASS
+# RUN: not llc -mtriple=amdgcn --passes='regallocfast<filter=bad-filter>' --print-pipeline-passes --filetype=null %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER
+
+# PASS: regallocfast<filter=sgpr>
+# BAD-FILTER: invalid regallocfast register filter 'bad-filter'
+
+---
+name: f
+...
diff --git a/llvm/test/tools/llvm-cov/gcov/intermediate-format.test b/llvm/test/tools/llvm-cov/gcov/intermediate-format.test
index 583e670c2d3fa38e40bb2f2d2d57489535729485..a3f4695c3204fa825be9720681fd63ea2e44e438 100644
--- a/llvm/test/tools/llvm-cov/gcov/intermediate-format.test
+++ b/llvm/test/tools/llvm-cov/gcov/intermediate-format.test
@@ -1,5 +1,3 @@
-REQUIRES: shell
-
 RUN: rm -rf %t && mkdir %t && cd %t
 RUN: cp %S/Inputs/test.gcno %S/Inputs/test.gcda .
 
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
index bd7a4894b45d6bb2c2f7d82be890cdf3dfcde730..7150a586fe6fd4bf1e6ff6beba252e7027f2ce95 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
@@ -1455,20 +1455,20 @@ vzeroupper
 # CHECK-NEXT:  3      2     1.00           *            vpextrq	$1, %xmm0, (%rax)
 # CHECK-NEXT:  2      3     1.00                        vpextrw	$1, %xmm0, %ecx
 # CHECK-NEXT:  3      2     1.00           *            vpextrw	$1, %xmm0, (%rax)
-# CHECK-NEXT:  3      3     2.00                        vphaddd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphaddd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  3      3     2.00                        vphaddsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphaddsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  3      3     2.00                        vphaddw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      4     1.00                        vphminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   vphminposuw	(%rax), %xmm2
-# CHECK-NEXT:  3      3     2.00                        vphsubd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphsubd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  3      3     2.00                        vphsubsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  3      3     2.00                        vphsubw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   vphsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vphsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     2.00                        vpinsrb	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     2.00                        vpinsrd	$1, %eax, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     126.00 322.92 237.92 160.50 160.50 19.00  291.92 6.25   19.00  19.00  19.00
+# CHECK-NEXT:  -     126.00 325.58 252.58 160.50 160.50 19.00  274.58 6.25   19.00  19.00  19.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -1908,22 +1908,22 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -     0.50    -      -     0.50   0.50   0.50   vextractf128	$1, %ymm0, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -      -      -     vextractps	$1, %xmm0, %ecx
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   1.00    -     0.50   0.50   0.50   vextractps	$1, %xmm0, (%rax)
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhaddpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhaddpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhaddpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhaddpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhaddps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhaddps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhaddps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhaddps	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhsubpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhsubpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhsubpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhsubpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhsubps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhsubps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vhsubps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vhsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vhsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vhsubps	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vinsertf128	$1, %xmm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -      -      -     vinsertf128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vinsertps	$1, %xmm0, %xmm1, %xmm2
@@ -2167,20 +2167,20 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   1.00    -     0.50   0.50   0.50   vpextrq	$1, %xmm0, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -      -      -     vpextrw	$1, %xmm0, %ecx
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   1.00    -     0.50   0.50   0.50   vpextrw	$1, %xmm0, (%rax)
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphaddd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphaddd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vphaddsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vphaddsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphaddw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     vphaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     vphaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphaddw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     vphminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -      -      -     vphminposuw	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphsubd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphsubd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vphsubsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vphsubsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphsubw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     vphsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     vphsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphsubw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrb	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrb	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrd	$1, %eax, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
index dcf883445ba4e9ac860d39440e352235cee72452..c251dc349587c19924fcce957f860d900ae2a36d 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
@@ -576,18 +576,18 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  5      20    2.00    *                   vpgatherqd	%xmm0, (%rax,%ymm1,2), %xmm2
 # CHECK-NEXT:  5      18    1.00    *                   vpgatherqq	%xmm0, (%rax,%xmm1,2), %xmm2
 # CHECK-NEXT:  5      20    2.00    *                   vpgatherqq	%ymm0, (%rax,%ymm1,2), %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphaddd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphaddd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphaddsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphaddsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphaddw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphaddw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphsubd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphsubd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphsubsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphsubsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  3      3     2.00                        vphsubw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  4      10    2.00    *                   vphsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vphsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphsubw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      12    0.50    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      5     0.50                        vpmaddwd	%ymm0, %ymm1, %ymm2
@@ -778,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     110.33 104.33 98.00  98.00  2.50   149.33  -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     110.33 116.33 98.00  98.00  2.50   137.33  -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -898,18 +898,18 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     1.33   0.33   2.00   2.00    -     1.33    -      -      -      -     vpgatherqd	%xmm0, (%rax,%ymm1,2), %xmm2
 # CHECK-NEXT:  -      -     1.33   0.33   1.00   1.00    -     1.33    -      -      -      -     vpgatherqq	%xmm0, (%rax,%xmm1,2), %xmm2
 # CHECK-NEXT:  -      -     1.33   0.33   2.00   2.00    -     1.33    -      -      -      -     vpgatherqq	%ymm0, (%rax,%ymm1,2), %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphaddd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphaddd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vphaddsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vphaddsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphaddw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphaddw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphsubd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphsubd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     vphsubsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     vphsubsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     vphsubw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     vphsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     vphaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     vphaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     vphsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     vphsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphsubw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpmaddubsw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpmaddubsw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpmaddwd	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s
index 4d1942450ec636da9d2155ffb5cf5d212a082431..0d075a9bc3bf52fd8c6500e05e0e971e4d8e3192 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s
@@ -81,7 +81,7 @@ mwait
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     6.67   7.67   5.00   5.00    -     23.67  4.00    -      -      -
+# CHECK-NEXT:  -      -     8.00   9.00   5.00   5.00    -     21.00  4.00    -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -89,14 +89,14 @@ mwait
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     addsubpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     addsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     addsubps	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     haddpd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     haddpd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     haddps	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     haddps	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     hsubpd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     hsubpd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     hsubps	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     hsubps	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     haddpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     haddpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     haddps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     haddps	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     hsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     hsubpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     hsubps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     lddqu	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -      -      -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     movddup	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s
index 3a6668cedb609aa93e22d017fa78177bdf4113af..d034cbd0f6398033081884f7429cc4784ea00af9 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s
@@ -124,28 +124,28 @@ psignw      (%rax), %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   palignr	$1, (%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phaddd	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phaddd	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phaddd	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phaddd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phaddd	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddd	(%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phaddsw	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phaddsw	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phaddsw	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phaddsw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phaddsw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddsw	(%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phaddw	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phaddw	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phaddw	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phaddw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phaddw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddw	(%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phsubd	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phsubd	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phsubd	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phsubd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phsubd	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubd	(%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phsubsw	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phsubsw	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phsubsw	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phsubsw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phsubsw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubsw	(%rax), %xmm2
 # CHECK-NEXT:  3      3     2.00                        phsubw	%mm0, %mm2
 # CHECK-NEXT:  4      8     2.00    *                   phsubw	(%rax), %mm2
-# CHECK-NEXT:  3      3     2.00                        phsubw	%xmm0, %xmm2
-# CHECK-NEXT:  4      9     2.00    *                   phsubw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phsubw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubw	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        pmaddubsw	%mm0, %mm2
 # CHECK-NEXT:  2      10    1.00    *                   pmaddubsw	(%rax), %mm2
 # CHECK-NEXT:  1      5     0.50                        pmaddubsw	%xmm0, %xmm2
@@ -187,7 +187,7 @@ psignw      (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     30.67  13.67  16.00  16.00   -     67.67   -      -      -      -
+# CHECK-NEXT:  -      -     30.67  25.67  16.00  16.00   -     55.67   -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -209,28 +209,28 @@ psignw      (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     palignr	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     2.50    -      -      -      -     phaddd	%mm0, %mm2
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     2.50    -      -      -      -     phaddd	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     phaddd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     phaddd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     phaddd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     phaddd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     2.00    -      -      -      -     phaddsw	%mm0, %mm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     2.00    -      -      -      -     phaddsw	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     phaddsw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     phaddsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     phaddsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     phaddsw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     2.50    -      -      -      -     phaddw	%mm0, %mm2
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     2.50    -      -      -      -     phaddw	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     phaddw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     phaddw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     phaddw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     phaddw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     2.50    -      -      -      -     phsubd	%mm0, %mm2
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     2.50    -      -      -      -     phsubd	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     phsubd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     phsubd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     phsubd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     phsubd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     2.00    -      -      -      -     phsubsw	%mm0, %mm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     2.00    -      -      -      -     phsubsw	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -      -      -     phsubsw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -      -      -     phsubsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50    -      -      -     1.00    -      -      -      -     phsubsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     phsubsw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     2.50    -      -      -      -     phsubw	%mm0, %mm2
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     2.50    -      -      -      -     phsubw	(%rax), %mm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -      -      -     phsubw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -      -      -     phsubw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     phsubw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     phsubw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     pmaddubsw	%mm0, %mm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -      -      -     pmaddubsw	(%rax), %mm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pmaddubsw	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
index cabb002b82412850a194b740f15ecfa273ed6f0d..f4904e4467ffe3547178850bf33b161941441b47 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
@@ -1736,7 +1736,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     126.00 325.25 202.25 173.83 173.83 38.00  326.25 7.25   11.33
+# CHECK-NEXT:  -     126.00 327.92 204.92 173.83 173.83 38.00  320.92 7.25   11.33
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1906,22 +1906,22 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   vextractf128	$1, %ymm0, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vextractps	$1, %xmm0, %ecx
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00   1.00    -     0.33   vextractps	$1, %xmm0, (%rax)
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhaddpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhaddpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhaddpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhaddpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhaddps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhaddps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhaddps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhaddps	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhsubpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhsubpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhsubpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhsubpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhsubps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhsubps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     vhsubps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     vhsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     vhsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     vhsubps	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vinsertf128	$1, %xmm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vinsertf128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vinsertps	$1, %xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
index e6bec1953fb84205b96a2d9a20df3fe20216c04b..0b6b0350838915589d30f8e382759c7d455bda4a 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
@@ -79,7 +79,7 @@ mwait
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     6.67   6.67   5.00   5.00    -     24.67  4.00    -
+# CHECK-NEXT:  -      -     8.00   8.00   5.00   5.00    -     22.00  4.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -87,14 +87,14 @@ mwait
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     addsubpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     addsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     addsubps	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     haddpd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     haddpd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     haddps	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     haddps	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     hsubpd	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     hsubpd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     hsubps	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     hsubps	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     haddpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     haddpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     haddps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     haddps	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     hsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     hsubpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     hsubps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     lddqu	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
index f1262c5a2fc2a2c896fe80c61b3c096bebf57552..f79f358e83a7c8fbb21a2e893ef82703a63f5a4f 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
@@ -28,6 +28,11 @@ define amdgpu_kernel void @test_kernel() {
 ; RUN: llvm-objdump -D -mllvm --amdhsa-code-object-version=6 %t.o > %t-detect.txt
 ; RUN: diff %t-specify.txt %t-detect.txt
 
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1152 -filetype=obj -O0 -o %t.o %s
+; RUN: llvm-objdump -D --arch-name=amdgcn --mcpu=gfx1152 %t.o > %t-specify.txt
+; RUN: llvm-objdump -D %t.o > %t-detect.txt
+; RUN: diff %t-specify.txt %t-detect.txt
+
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 -filetype=obj -O0 -o %t.o %s
 ; RUN: llvm-objdump -D --arch-name=amdgcn --mcpu=gfx1151 %t.o > %t-specify.txt
 ; RUN: llvm-objdump -D %t.o > %t-detect.txt
diff --git a/llvm/test/tools/llvm-rc/windres-prefix.test b/llvm/test/tools/llvm-rc/windres-prefix.test
index 4c53fdfc3db65e328e538b943a6cb4980b04cc86..7dda51d0635232fa9b4f7bf870615287861ff91b 100644
--- a/llvm/test/tools/llvm-rc/windres-prefix.test
+++ b/llvm/test/tools/llvm-rc/windres-prefix.test
@@ -1,5 +1,3 @@
-; REQUIRES: shell
-
 ; RUN: rm -rf %t && mkdir %t
 
 ; Check that a triple prefix on the executable gets picked up as target triple.
diff --git a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
index f9e90e21400ea45c8ce9f3c46309454ccd3155ae..50d437b50b69b2b9ebf52a4e70adecb158af53c1 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
@@ -421,6 +421,15 @@
 # RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1151
 # RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1151 -DFLAG_VALUE=0x4A
 
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55
+
 # RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1200
 # RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1200 -DFLAG_VALUE=0x48
 
diff --git a/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll b/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll
index 1ceeca8b65615626c809c7da43e165c0d30a744d..127543c64c3308e93b67fa2fdbf6f19fa08c041d 100644
--- a/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll
+++ b/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll
@@ -2,7 +2,7 @@
 ; DICompileUnit and DISuprogram.
 ;
 ; RUN: llvm-reduce --delta-passes=di-metadata --abort-on-invalid-reduction --test FileCheck --test-arg --check-prefixes=CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
-; RUN: FileCheck <%t --enable-var-scope %s
+; RUN: FileCheck <%t --enable-var-scope %s --implicit-check-not=DIGlobalVariableExpression
 
 ; CHECK-INTERESTINGNESS: define void @test() !dbg [[SUBPROG:![0-9]+]]
 ; CHECK-INTERESTINGNESS: !llvm.module.flags = !{
@@ -21,12 +21,10 @@
 
 ; CHECK: !llvm.dbg.cu = !{[[CU:.+]]}
 
-; CHECK-DAG: [[CU]] = distinct !DICompileUnit(language: DW_LANG_C99,{{.*}}, retainedTypes: [[TYPES:![0-9]+]], globals: [[GLOBALS:![0-9]+]]
-; CHECK-DAG: [[EMPTY:![0-9]+]] = !{}
+; CHECK-DAG: [[CU]] = distinct !DICompileUnit(language: DW_LANG_C99,{{.*}}, retainedTypes: [[TYPES:![0-9]+]], globals: [[EMPTY:![0-9]+]]
+; CHECK-DAG: [[EMPTY]] = !{}
 ; CHECK-DAG: [[TYPES]] = !{[[T0:![0-9]+]]
 ; CHECK-DAG: [[T0]] = !DIBasicType(name: "unsigned int",
-; CHECK-DAG: [[GLOBALS]] = !{{{![0-9]+}}
-
 ; CHECK-DAG: [[SUBPROG]] = distinct !DISubprogram(name: "test", {{.*}}retainedNodes: [[EMPTY]])
 
 define void @test() !dbg !17 {
diff --git a/llvm/test/tools/split-file/output-is-special.test b/llvm/test/tools/split-file/output-is-special.test
index 98bb4d36a4ff314288ab479edf9414d716a52607..0b1e0f786c4d2b667a8c21055f19b5c18d7df6d1 100644
--- a/llvm/test/tools/split-file/output-is-special.test
+++ b/llvm/test/tools/split-file/output-is-special.test
@@ -1,5 +1,4 @@
 # UNSUPPORTED: system-windows
-# REQUIRES: shell
 
 ## Don't delete the output if it is special, otherwise root may accidentally
 ## remove important special files.
diff --git a/llvm/tools/llvm-as/llvm-as.cpp b/llvm/tools/llvm-as/llvm-as.cpp
index e48e3f4d22c1231941ec11a65b9d6013ce699030..0958e16c2197acaf56dfd15f3eaa27698ec2ad4a 100644
--- a/llvm/tools/llvm-as/llvm-as.cpp
+++ b/llvm/tools/llvm-as/llvm-as.cpp
@@ -142,11 +142,10 @@ int main(int argc, char **argv) {
   }
 
   // Convert to new debug format if requested.
-  assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug mode");
-  if (UseNewDbgInfoFormat && WriteNewDbgInfoFormatToBitcode) {
-    M->convertToNewDbgValues();
+  M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat &&
+                           WriteNewDbgInfoFormatToBitcode);
+  if (M->IsNewDbgInfoFormat)
     M->removeDebugIntrinsicDeclarations();
-  }
 
   std::unique_ptr<ModuleSummaryIndex> Index = std::move(ModuleAndIndex.Index);
 
diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c
index 35c65f885af329e4d9490d18e468a6609dbf4c4e..032657203b5f2ee7bb5fdadf8c6405e3585761bc 100644
--- a/llvm/tools/llvm-c-test/debuginfo.c
+++ b/llvm/tools/llvm-c-test/debuginfo.c
@@ -228,7 +228,24 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) {
   LLVMPositionBuilderAtEnd(Builder, FooVarBlock);
   LLVMTypeRef I64 = LLVMInt64TypeInContext(Ctx);
   LLVMValueRef Zero = LLVMConstInt(I64, 0, false);
-  LLVMBuildRet(Builder, Zero);
+  LLVMValueRef Ret = LLVMBuildRet(Builder, Zero);
+
+  // Insert a `phi` before the `ret`. In the new debug info mode we need to
+  // be careful to insert before debug records too, else the debug records
+  // will come before the `phi` (and be absorbed onto it) which is an invalid
+  // state.
+  LLVMValueRef InsertPos = LLVMGetFirstInstruction(FooVarBlock);
+  LLVMPositionBuilderBeforeInstrAndDbgRecords(Builder, InsertPos);
+  LLVMValueRef Phi1 = LLVMBuildPhi(Builder, I64, "p1");
+  LLVMAddIncoming(Phi1, &Zero, &FooEntryBlock, 1);
+  // Do the same again using the other position-setting function.
+  LLVMPositionBuilderBeforeDbgRecords(Builder, FooVarBlock, InsertPos);
+  LLVMValueRef Phi2 = LLVMBuildPhi(Builder, I64, "p2");
+  LLVMAddIncoming(Phi2, &Zero, &FooEntryBlock, 1);
+  // Insert a non-phi before the `ret` but not before the debug records to
+  // test that works as expected.
+  LLVMPositionBuilder(Builder, FooVarBlock, Ret);
+  LLVMBuildAdd(Builder, Phi1, Phi2, "a");
 
   char *MStr = LLVMPrintModuleToString(M);
   puts(MStr);
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index fbbb5506e43e0530a0c4f7901d22cbc891aac6a4..d28af85bc739eb4960e7ac9871f448c9dd91563c 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -258,7 +258,7 @@ int main(int argc, char **argv) {
       // All that llvm-dis does is write the assembly to a file.
       if (!DontPrint) {
         if (M) {
-          ScopedDbgInfoFormatSetter FormatSetter(*M, WriteNewDbgInfoFormat);
+          M->setIsNewDbgInfoFormat(WriteNewDbgInfoFormat);
           if (WriteNewDbgInfoFormat)
             M->removeDebugIntrinsicDeclarations();
           M->print(Out->os(), Annotator.get(), PreserveAssemblyUseListOrder);
diff --git a/llvm/tools/llvm-jitlink/CMakeLists.txt b/llvm/tools/llvm-jitlink/CMakeLists.txt
index c9d1d23d3471a65325c7d8b8d40cfd62a4b4c5f5..1487e7897f600149305ca41874b87d26f1b27d6b 100644
--- a/llvm/tools/llvm-jitlink/CMakeLists.txt
+++ b/llvm/tools/llvm-jitlink/CMakeLists.txt
@@ -28,8 +28,12 @@ add_llvm_tool(llvm-jitlink
   llvm-jitlink-statistics.cpp
   )
 
- if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Haiku")
+  target_link_libraries(llvm-jitlink PRIVATE network)
+endif()
+
+if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
   target_link_libraries(llvm-jitlink PRIVATE socket)
- endif()
+endif()
 
 export_executable_symbols(llvm-jitlink)
diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index 7794f2d81ed064d234b11914a8fcbda79edddb96..b84469d1c757f8da9cca2bda04b9afb6ee1f119c 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -489,12 +489,6 @@ int main(int argc, char **argv) {
   if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET)
     LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE;
 
-  // RemoveDIs debug-info transition: tests may request that we /try/ to use the
-  // new debug-info format.
-  if (TryUseNewDbgInfoFormat) {
-    // Turn the new debug-info format on.
-    UseNewDbgInfoFormat = true;
-  }
   // Since llvm-link collects multiple IR modules together, for simplicity's
   // sake we disable the "PreserveInputDbgFormat" flag to enforce a single
   // debug info format.
@@ -556,7 +550,7 @@ int main(int argc, char **argv) {
     SetFormat(WriteNewDbgInfoFormat);
     Composite->print(Out.os(), nullptr, PreserveAssemblyUseListOrder);
   } else if (Force || !CheckBitcodeOutputToConsole(Out.os())) {
-    SetFormat(WriteNewDbgInfoFormatToBitcode);
+    SetFormat(UseNewDbgInfoFormat && WriteNewDbgInfoFormatToBitcode);
     WriteBitcodeToFile(*Composite, Out.os(), PreserveBitcodeUseListOrder);
   }
 
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 40494da8b21bda74a61b59312cb17679d868b08e..c696934a959b2a63d1c83a544edcfba8c146c735 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1631,6 +1631,7 @@ const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1103, "gfx1103"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1150, "gfx1150"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"),                          \
+  ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1152, "gfx1152"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, "gfx9-generic"),                \
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp
index f4d8496aba4a7c7e94bb402e58e75318b935ea3f..38352d6342d4f9a456aacefbb85f1dea75fe2248 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp
@@ -65,12 +65,13 @@ void identifyUninterestingMDNodes(Oracle &O, MDNodeList &MDs) {
     SmallVector<Metadata *, 16> TN;
     for (size_t I = 0; I < Tup->getNumOperands(); ++I) {
       // Ignore any operands that are not DebugInfo metadata nodes.
-      if (isa_and_nonnull<DINode>(Tup->getOperand(I)))
-        // Don't add uninteresting operands to the tuple.
-        if (!O.shouldKeep())
-          continue;
-
-      TN.push_back(Tup->getOperand(I));
+      if (Metadata *Op = Tup->getOperand(I).get()) {
+        if (isa<DINode>(Op) || isa<DIGlobalVariableExpression>(Op))
+          // Don't add uninteresting operands to the tuple.
+          if (!O.shouldKeep())
+            continue;
+        TN.push_back(Op);
+      }
     }
     if (TN.size() != Tup->getNumOperands())
       DbgNode->replaceOperandWith(OpIdx, DbgNode->get(DbgNode->getContext(), TN));
diff --git a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
index f6a053792f85298d3d5237c2fedefbf72ab3a23d..24f4f11db9a8b9c8b379635f7c0033fde2a86262 100644
--- a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
+++ b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/IRSimilarityIdentifier.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -22,6 +23,11 @@
 using namespace llvm;
 using namespace IRSimilarity;
 
+extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
+extern cl::opt<cl::boolOrDefault> PreserveInputDbgFormat;
+extern bool WriteNewDbgInfoFormatToBitcode;
+extern cl::opt<bool> WriteNewDbgInfoFormat;
+
 static std::unique_ptr<Module> makeLLVMModule(LLVMContext &Context,
                                               StringRef ModuleStr) {
   SMDiagnostic Err;
@@ -1306,19 +1312,18 @@ TEST(IRInstructionMapper, CallBrInstIllegal) {
   ASSERT_GT(UnsignedVec[0], Mapper.IllegalInstrNumber);
 }
 
-// Checks that an debuginfo intrinsics are mapped to be invisible.  Since they
+// Checks that an debuginfo records are mapped to be invisible. Since they
 // do not semantically change the program, they can be recognized as similar.
 TEST(IRInstructionMapper, DebugInfoInvisible) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           then:
-                            %0 = add i32 %a, %b                    
-                            call void @llvm.dbg.value(metadata !0)
-                            %1 = add i32 %a, %b     
+                            %0 = add i32 %a, %b
+                              #dbg_value(i32 0, !0, !0, !0)
+                            %1 = add i32 %a, %b
                             ret i32 0
                           }
 
-                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10})";
   LLVMContext Context;
   std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleString);
@@ -1914,19 +1919,19 @@ TEST(IRSimilarityCandidate, CheckRegionsDifferentTypes) {
   ASSERT_FALSE(longSimCandCompare(InstrList));
 }
 
-// Check that debug instructions do not impact similarity. They are marked as
+// Check that debug records do not impact similarity. They are marked as
 // invisible.
 TEST(IRSimilarityCandidate, IdenticalWithDebug) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           bb0:
                              %0 = add i32 %a, %b
-                             call void @llvm.dbg.value(metadata !0)
+                               #dbg_value(i32 0, !0, !0, !0)
                              %1 = add i32 %b, %a
                              ret i32 0
                           bb1:
                              %2 = add i32 %a, %b
-                             call void @llvm.dbg.value(metadata !1)
+                               #dbg_value(i32 1, !1, !1, !1)
                              %3 = add i32 %b, %a
                              ret i32 0
                           bb2:
@@ -1935,7 +1940,6 @@ TEST(IRSimilarityCandidate, IdenticalWithDebug) {
                              ret i32 0       
                           }
 
-                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10}
                           !1 = distinct !{!"test\00", i32 11})";
   LLVMContext Context;
diff --git a/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp b/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
index 95f17f93bf4c0b85020834ee277b239b8a3290b8..d97549cb69d556dd83c29ac945ca65eecbf49ba6 100644
--- a/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
+++ b/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
@@ -99,7 +99,7 @@ TEST_F(ReplaceWithVecLibTest, TestValidMapping) {
                        ElementCount::getScalable(4), /*Masked*/ true,
                        "_ZGVsMxvu"};
   EXPECT_EQ(run(CorrectVD, IR),
-            "Instructions replaced with vector libraries: 1");
+            "Intrinsic calls replaced with vector libraries: 1");
 }
 
 // The VFABI prefix in TLI describes signature which is not matching the powi
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
index 7148e2902fa7ab1a902253b232a094ef2766dafd..ca50187e5e5ee02653d95cd541c54c724b1b9697 100644
--- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -63,6 +63,11 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
                                     "riscv64"),
             "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
 
+  // Check that LoongArch64 upgrades -n64 to -n32:64.
+  EXPECT_EQ(UpgradeDataLayoutString("e-m:e-p:64:64-i64:64-i128:128-n64-S128",
+                                    "loongarch64"),
+            "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
+
   // Check that SPIR && SPIRV targets add -G1 if it's not present.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir"), "e-p:32:32-G1");
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir64"), "e-p:32:32-G1");
diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index 53a74c833eb319ab387e265c44dd8e418f299603..39a49eb1799936d4351644ce60e5149b7b804339 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -24,6 +24,32 @@ class CoreAPIsStandardTest : public CoreAPIsBasedStandardTest {};
 
 namespace {
 
+class CustomError : public ErrorInfo<CustomError> {
+public:
+  static char ID;
+  void log(raw_ostream &OS) const override { OS << "CustomError"; }
+  std::error_code convertToErrorCode() const override { return {}; }
+};
+char CustomError::ID = 0;
+
+TEST_F(CoreAPIsStandardTest, ErrorReporter) {
+  // Check that errors reported via ExecutionSession::reportError are sent to
+  // the registered error reporter, and that the error reporter can hold
+  // uniquely owned state.
+
+  Error ReportedError = Error::success();
+
+  ES.setErrorReporter(
+      // Make sure error reporter can capture uniquely-owned state.
+      [&, State = std::make_unique<int>(42)](Error Err) {
+        ReportedError = joinErrors(std::move(Err), std::move(ReportedError));
+      });
+
+  ES.reportError(make_error<CustomError>());
+
+  EXPECT_THAT_ERROR(std::move(ReportedError), Failed<CustomError>());
+}
+
 TEST_F(CoreAPIsStandardTest, JITDylibAddToLinkOrder) {
   // Check that the JITDylib::addToLinkOrder methods behave as expected.
   auto &JD2 = ES.createBareJITDylib("JD2");
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index f873bbd4293af5f496347719943e6e505239a6c9..91a0745a0cc76e48ef4b518e4ac5da48f637a979 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -25,8 +25,6 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
   SMDiagnostic Err;
   std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
@@ -44,8 +42,6 @@ namespace {
 // by DbgVariableRecords, the dbg.value replacement.
 TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -72,8 +68,6 @@ TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
-  // Convert the module to "new" form debug-info.
-  M->convertToNewDbgValues();
   // Fetch the entry block.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
 
@@ -103,16 +97,10 @@ TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
   EXPECT_TRUE(RetInst->hasDbgRecords());
   auto Range2 = RetInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(Range2.begin(), Range2.end()), 1u);
-
-  M->convertFromNewDbgValues();
-
-  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"---(
     define dso_local void @func() #0 !dbg !10 {
       %1 = alloca i32, align 4
@@ -150,8 +138,6 @@ TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   )---");
   ASSERT_TRUE(M);
 
-  M->convertToNewDbgValues();
-
   Function *F = M->getFunction("func");
 
   BasicBlock &BB = F->getEntryBlock();
@@ -161,14 +147,10 @@ TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   BasicBlock &BBBefore = F->getEntryBlock();
   auto I2 = std::prev(BBBefore.end(), 2);
   ASSERT_TRUE(I2->hasDbgRecords());
-
-  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, MarkerOperations) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -196,8 +178,6 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) {
 
   // Fetch the entry block,
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
-  // Convert the module to "new" form debug-info.
-  M->convertToNewDbgValues();
   EXPECT_EQ(BB.size(), 2u);
 
   // Fetch out our two markers,
@@ -295,14 +275,10 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) {
 
   // Teardown,
   Instr1->insertBefore(BB, BB.begin());
-
-  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -332,8 +308,6 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
   // Test that the movement of debug-data when using moveBefore etc and
   // insertBefore etc are governed by the "head" bit of iterators.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
-  // Convert the module to "new" form debug-info.
-  M->convertToNewDbgValues();
 
   // Test that the head bit behaves as expected: it should be set when the
   // code wants the _start_ of the block, but not otherwise.
@@ -404,14 +378,10 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
               DInst->DebugMarker->StoredDbgRecords.empty());
   EXPECT_FALSE(CInst->DebugMarker->StoredDbgRecords.empty());
   EXPECT_EQ(&*BB.begin(), CInst);
-
-  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -441,8 +411,6 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   // Check that DbgVariableRecords can be accessed from Instructions without
   // digging into the depths of DbgMarkers.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
-  // Convert the module to "new" form debug-info.
-  M->convertToNewDbgValues();
 
   Instruction *BInst = &*BB.begin();
   Instruction *CInst = BInst->getNextNode();
@@ -483,8 +451,6 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   CInst->dropOneDbgRecord(DVR1);
   EXPECT_FALSE(CInst->hasDbgRecords());
   EXPECT_EQ(CInst->DebugMarker->StoredDbgRecords.size(), 0u);
-
-  UseNewDbgInfoFormat = false;
 }
 
 /* Let's recall the big illustration from BasicBlock::spliceDebugInfo:
@@ -577,9 +543,7 @@ protected:
   DbgVariableRecord *DVRA, *DVRB, *DVRConst;
 
   void SetUp() override {
-    UseNewDbgInfoFormat = true;
     M = parseIR(C, SpliceTestIR.c_str());
-    M->convertToNewDbgValues();
 
     BBEntry = &M->getFunction("f")->getEntryBlock();
     BBExit = BBEntry->getNextNode();
@@ -599,8 +563,6 @@ protected:
         cast<DbgVariableRecord>(&*CInst->DebugMarker->StoredDbgRecords.begin());
   }
 
-  void TearDown() override { UseNewDbgInfoFormat = false; }
-
   bool InstContainsDbgVariableRecord(Instruction *I, DbgVariableRecord *DVR) {
     for (DbgRecord &D : I->getDbgRecordRange()) {
       if (&D == DVR) {
@@ -1187,8 +1149,6 @@ metadata !9, metadata !DIExpression()), !dbg !11 Dest      %c = add i16 %b, 1,
 // then the trailing DbgVariableRecords should get flushed back out.
 TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1219,7 +1179,6 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
-  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1234,8 +1193,6 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
   Instruction *BInst = &*Entry.begin();
   ASSERT_TRUE(BInst->DebugMarker);
   EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 1u);
-
-  UseNewDbgInfoFormat = false;
 }
 
 // When we remove instructions from the program, adjacent DbgVariableRecords
@@ -1244,8 +1201,6 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
 // dbg.values. Test that this can be replicated correctly by DbgVariableRecords
 TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1273,7 +1228,6 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
 )");
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
-  M->convertToNewDbgValues();
 
   // Fetch the relevant instructions from the converted function.
   Instruction *SubInst = &*Entry.begin();
@@ -1316,16 +1270,12 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
   EXPECT_EQ(std::distance(R4.begin(), R4.end()), 1u);
   auto R5 = RetInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(R5.begin(), R5.end()), 1u);
-
-  UseNewDbgInfoFormat = false;
 }
 
 // Test instruction removal and re-insertion, this time with one
 // DbgVariableRecord that should hop up one instruction.
 TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1352,7 +1302,6 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
 )");
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
-  M->convertToNewDbgValues();
 
   // Fetch the relevant instructions from the converted function.
   Instruction *SubInst = &*Entry.begin();
@@ -1391,8 +1340,6 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
   EXPECT_FALSE(RetInst->hasDbgRecords());
   auto R3 = AddInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(R3.begin(), R3.end()), 1u);
-
-  UseNewDbgInfoFormat = false;
 }
 
 // Similar to the above, what if we splice into an empty block with debug-info,
@@ -1401,8 +1348,6 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
 // of the i16 0 dbg.value.
 TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1436,7 +1381,6 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
-  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1463,16 +1407,12 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
 
   // No trailing DbgVariableRecords in the entry block now.
   EXPECT_EQ(Entry.getTrailingDbgRecords(), nullptr);
-
-  UseNewDbgInfoFormat = false;
 }
 
 // Similar test again, but this time: splice the contents of exit into entry,
 // with the intention of leaving the first dbg.value (i16 0) behind.
 TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1506,7 +1446,6 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
-  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1537,16 +1476,12 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   EXPECT_FALSE(Exit.getTrailingDbgRecords()->empty());
   Exit.getTrailingDbgRecords()->eraseFromParent();
   Exit.deleteTrailingDbgRecords();
-
-  UseNewDbgInfoFormat = false;
 }
 
 // What if we moveBefore end() -- there might be no debug-info there, in which
 // case we shouldn't crash.
 TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   LLVMContext C;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1576,7 +1511,6 @@ TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
-  M->convertToNewDbgValues();
 
   // Move the return to the end of the entry block.
   Instruction *Br = Entry.getTerminator();
@@ -1589,8 +1523,6 @@ TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   EXPECT_EQ(Entry.getTrailingDbgRecords(), nullptr);
   EXPECT_EQ(Exit.getTrailingDbgRecords(), nullptr);
   EXPECT_FALSE(Ret->hasDbgRecords());
-
-  UseNewDbgInfoFormat = false;
 }
 
 } // End anonymous namespace.
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index ec3f33318f8cdb4a1aa30fddb1defbb47a37852e..cac8acbe15a79da238756b5972f1b0b0a608bfa0 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -156,7 +156,7 @@ TEST(StripTest, LoopMetadata) {
   EXPECT_FALSE(BrokenDebugInfo);
 }
 
-TEST(MetadataTest, DeleteInstUsedByDbgValue) {
+TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -187,12 +187,13 @@ TEST(MetadataTest, DeleteInstUsedByDbgValue) {
 
   // Find the dbg.value using %b.
   SmallVector<DbgValueInst *, 1> DVIs;
-  findDbgValues(DVIs, &I);
+  SmallVector<DbgVariableRecord *, 1> DVRs;
+  findDbgValues(DVIs, &I, &DVRs);
 
   // Delete %b. The dbg.value should now point to undef.
   I.eraseFromParent();
-  EXPECT_EQ(DVIs[0]->getNumVariableLocationOps(), 1u);
-  EXPECT_TRUE(isa<UndefValue>(DVIs[0]->getValue(0)));
+  EXPECT_EQ(DVRs[0]->getNumVariableLocationOps(), 1u);
+  EXPECT_TRUE(isa<UndefValue>(DVRs[0]->getValue(0)));
 }
 
 TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
@@ -230,8 +231,8 @@ TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
 
   // Get the dbg.declare.
   Function &F = *cast<Function>(M->getNamedValue("fun"));
-  DbgVariableIntrinsic *DbgDeclare =
-      cast<DbgVariableIntrinsic>(&F.front().front());
+  DbgVariableRecord *DbgDeclare =
+      cast<DbgVariableRecord>(&*F.front().front().getDbgRecordRange().begin());
   // Check that this form counts as a "no location" marker.
   EXPECT_TRUE(DbgDeclare->isKillLocation());
 }
@@ -239,6 +240,9 @@ TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
 // Duplicate of above test, but in DbgVariableRecord representation.
 TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   LLVMContext C;
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -264,10 +268,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
   Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHI();
-  M->convertToNewDbgValues();
 
   // Find the DbgVariableRecords using %b.
   SmallVector<DbgValueInst *, 2> DVIs;
@@ -289,6 +290,8 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
 // Ensure that the order of dbg.value intrinsics returned by findDbgValues, and
 // their corresponding DbgVariableRecord representation, are consistent.
 TEST(MetadataTest, OrderingOfDbgVariableRecords) {
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = false;
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -316,8 +319,6 @@ TEST(MetadataTest, OrderingOfDbgVariableRecords) {
     !12 = !DILocalVariable(name: "bar", scope: !6, file: !1, line: 1, type: !10)
 )");
 
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
   Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHI();
 
   SmallVector<DbgValueInst *, 2> DVIs;
@@ -515,14 +516,15 @@ TEST(DbgAssignIntrinsicTest, replaceVariableLocationOp) {
   Value *V1 = Fun.getArg(0);
   Value *P1 = Fun.getArg(1);
   Value *P2 = Fun.getArg(2);
-  DbgAssignIntrinsic *DAI = cast<DbgAssignIntrinsic>(Fun.begin()->begin());
-  ASSERT_TRUE(V1 == DAI->getVariableLocationOp(0));
-  ASSERT_TRUE(P1 == DAI->getAddress());
+  DbgVariableRecord *DbgAssign = cast<DbgVariableRecord>(
+      &*Fun.front().front().getDbgRecordRange().begin());
+  ASSERT_TRUE(V1 == DbgAssign->getVariableLocationOp(0));
+  ASSERT_TRUE(P1 == DbgAssign->getAddress());
 
 #define TEST_REPLACE(Old, New, ExpectedValue, ExpectedAddr)                    \
-  DAI->replaceVariableLocationOp(Old, New);                                    \
-  EXPECT_EQ(DAI->getVariableLocationOp(0), ExpectedValue);                     \
-  EXPECT_EQ(DAI->getAddress(), ExpectedAddr);
+  DbgAssign->replaceVariableLocationOp(Old, New);                              \
+  EXPECT_EQ(DbgAssign->getVariableLocationOp(0), ExpectedValue);               \
+  EXPECT_EQ(DbgAssign->getAddress(), ExpectedAddr);
 
   // Replace address only.
   TEST_REPLACE(/*Old*/ P1, /*New*/ P2, /*Value*/ V1, /*Address*/ P2);
@@ -533,8 +535,8 @@ TEST(DbgAssignIntrinsicTest, replaceVariableLocationOp) {
 
   // Replace address only, value uses a DIArgList.
   // Value = {DIArgList(V1)}, Addr = P1.
-  DAI->setRawLocation(DIArgList::get(C, ValueAsMetadata::get(V1)));
-  DAI->setExpression(DIExpression::get(
+  DbgAssign->setRawLocation(DIArgList::get(C, ValueAsMetadata::get(V1)));
+  DbgAssign->setExpression(DIExpression::get(
       C, {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_stack_value}));
   TEST_REPLACE(/*Old*/ P1, /*New*/ P2, /*Value*/ V1, /*Address*/ P2);
 #undef TEST_REPLACE
@@ -620,11 +622,11 @@ TEST(AssignmentTrackingTest, Utils) {
   //
   // Check there are two llvm.dbg.assign intrinsics linked to Alloca.
   auto CheckFun1Mapping = [&Alloca]() {
-    auto Markers = at::getAssignmentMarkers(&Alloca);
+    auto Markers = at::getDVRAssignmentMarkers(&Alloca);
     EXPECT_TRUE(std::distance(Markers.begin(), Markers.end()) == 2);
     // Check those two entries are distinct.
-    DbgAssignIntrinsic *First = *Markers.begin();
-    DbgAssignIntrinsic *Second = *std::next(Markers.begin());
+    DbgVariableRecord *First = *Markers.begin();
+    DbgVariableRecord *Second = *std::next(Markers.begin());
     EXPECT_NE(First, Second);
 
     // Check that we can get back to Alloca from each llvm.dbg.assign.
@@ -660,7 +662,7 @@ TEST(AssignmentTrackingTest, Utils) {
   DIAssignID *Fun2ID = cast_or_null<DIAssignID>(
       Fun2Alloca.getMetadata(LLVMContext::MD_DIAssignID));
   EXPECT_NE(New, Fun2ID);
-  auto Fun2Markers = at::getAssignmentMarkers(&Fun2Alloca);
+  auto Fun2Markers = at::getDVRAssignmentMarkers(&Fun2Alloca);
   ASSERT_TRUE(std::distance(Fun2Markers.begin(), Fun2Markers.end()) == 1);
   auto Fun2Insts = at::getAssignmentInsts(*Fun2Markers.begin());
   ASSERT_TRUE(std::distance(Fun2Insts.begin(), Fun2Insts.end()) == 1);
@@ -669,10 +671,10 @@ TEST(AssignmentTrackingTest, Utils) {
   // 3. Check that deleting dbg.assigns from a specific instruction works.
   Instruction &Fun3Alloca =
       *M->getFunction("fun3")->getEntryBlock().getFirstNonPHIOrDbg();
-  auto Fun3Markers = at::getAssignmentMarkers(&Fun3Alloca);
+  auto Fun3Markers = at::getDVRAssignmentMarkers(&Fun3Alloca);
   ASSERT_TRUE(std::distance(Fun3Markers.begin(), Fun3Markers.end()) == 1);
   at::deleteAssignmentMarkers(&Fun3Alloca);
-  Fun3Markers = at::getAssignmentMarkers(&Fun3Alloca);
+  Fun3Markers = at::getDVRAssignmentMarkers(&Fun3Alloca);
   EXPECT_EQ(Fun3Markers.empty(), true);
 
   // 4. Check that deleting works and applies only to the target function.
@@ -683,7 +685,7 @@ TEST(AssignmentTrackingTest, Utils) {
   // llvm.dbg.assign.
   EXPECT_EQ(Fun2ID, cast_or_null<DIAssignID>(
                         Fun2Alloca.getMetadata(LLVMContext::MD_DIAssignID)));
-  EXPECT_FALSE(at::getAssignmentMarkers(&Fun2Alloca).empty());
+  EXPECT_FALSE(at::getDVRAssignmentMarkers(&Fun2Alloca).empty());
 }
 
 TEST(IRBuilder, GetSetInsertionPointWithEmptyBasicBlock) {
@@ -769,12 +771,12 @@ TEST(AssignmentTrackingTest, InstrMethods) {
   // Use SetVectors to check that the attachments and markers are unique
   // (another test requirement).
   SetVector<Metadata *> OrigIDs;
-  SetVector<DbgAssignIntrinsic *> Markers;
+  SetVector<DbgVariableRecord *> Markers;
   for (const Instruction *SI : Stores) {
     Metadata *ID = SI->getMetadata(LLVMContext::MD_DIAssignID);
     ASSERT_TRUE(OrigIDs.insert(ID));
     ASSERT_TRUE(ID != nullptr);
-    auto Range = at::getAssignmentMarkers(SI);
+    auto Range = at::getDVRAssignmentMarkers(SI);
     ASSERT_TRUE(std::distance(Range.begin(), Range.end()) == 1);
     ASSERT_TRUE(Markers.insert(*Range.begin()));
   }
@@ -867,6 +869,8 @@ TEST(AssignmentTrackingTest, InstrMethods) {
 // dbg.values that have been converted to a non-instruction format.
 TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   LLVMContext C;
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -1041,14 +1045,14 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   // The record of those trailing DbgVariableRecords would dangle and cause an
   // assertion failure if it lived until the end of the LLVMContext.
   ExitBlock->deleteTrailingDbgRecords();
+  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   LLVMContext C;
 
-  // For the purpose of this test, set and un-set the command line option
-  // corresponding to UseNewDbgInfoFormat.
-  UseNewDbgInfoFormat = true;
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = false;
 
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -1079,6 +1083,11 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
+  // For the purpose of this test, set and un-set the command line option
+  // corresponding to UseNewDbgInfoFormat, but only after parsing, to ensure
+  // that the IR starts off in the old format.
+  UseNewDbgInfoFormat = true;
+
   // Check that the conversion routines and utilities between dbg.value
   // debug-info format and DbgVariableRecords works.
   Function *F = M->getFunction("f");
@@ -1183,7 +1192,7 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   EXPECT_EQ(DVI2->getVariable(), DLV2);
   EXPECT_EQ(DVI2->getExpression(), Expr2);
 
-  UseNewDbgInfoFormat = false;
+  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 // Test that the hashing function for DISubprograms representing methods produce
diff --git a/llvm/unittests/IR/DemandedBitsTest.cpp b/llvm/unittests/IR/DemandedBitsTest.cpp
index 4d15e81899612f8d12b35ffcca966b9c232060bd..b52173412a99a3ef7484b214621fe70aa254f559 100644
--- a/llvm/unittests/IR/DemandedBitsTest.cpp
+++ b/llvm/unittests/IR/DemandedBitsTest.cpp
@@ -38,8 +38,10 @@ static void TestBinOpExhaustive(Fn1 PropagateFn, Fn2 EvalFn) {
 
           APInt AB1R = PropagateFn(0, AOut, Known1Redacted, Known2Redacted);
           APInt AB2R = PropagateFn(1, AOut, Known1Redacted, Known2Redacted);
-          EXPECT_EQ(AB1, AB1R);
-          EXPECT_EQ(AB2, AB2R);
+          if (!Known1Redacted.hasConflict() && !Known2Redacted.hasConflict()) {
+            EXPECT_EQ(AB1, AB1R);
+            EXPECT_EQ(AB2, AB2R);
+          }
         }
         ForeachNumInKnownBits(Known1, [&](APInt Value1) {
           ForeachNumInKnownBits(Known2, [&](APInt Value2) {
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 2001df090aed53b4480f025ba1986c7bd6a63720..ff96df85812002249b012fb66c331175d69156cf 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -994,17 +994,17 @@ TEST_F(IRBuilderTest, DIBuilder) {
     EXPECT_TRUE(verifyModule(*M));
   };
 
-  // Test in old-debug mode.
-  EXPECT_FALSE(M->IsNewDbgInfoFormat);
+  // Test in new-debug mode.
+  EXPECT_TRUE(M->IsNewDbgInfoFormat);
   RunTest();
 
-  // Test in new-debug mode.
-  // Reset the test then call convertToNewDbgValues to flip the flag
+  // Test in old-debug mode.
+  // Reset the test then call convertFromNewDbgValues to flip the flag
   // on the test's Module, Function and BasicBlock.
   TearDown();
   SetUp();
-  M->convertToNewDbgValues();
-  EXPECT_TRUE(M->IsNewDbgInfoFormat);
+  M->convertFromNewDbgValues();
+  EXPECT_FALSE(M->IsNewDbgInfoFormat);
   RunTest();
 }
 
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index b47c73f0b329ae803c68368c7cb99e1b8e9a9aea..b6044b28629204c8d1028e3d4c8a1069897f837a 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -25,12 +25,15 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NoFolder.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm-c/Core.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
 #include <memory>
 
+extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
+
 namespace llvm {
 namespace {
 
@@ -1460,6 +1463,8 @@ TEST(InstructionsTest, GetSplat) {
 
 TEST(InstructionsTest, SkipDebug) {
   LLVMContext C;
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C,
                                       R"(
       declare void @llvm.dbg.value(metadata, metadata, metadata)
@@ -1495,6 +1500,7 @@ TEST(InstructionsTest, SkipDebug) {
 
   // After the terminator, there are no non-debug instructions.
   EXPECT_EQ(nullptr, Term->getNextNonDebugInstruction());
+  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(InstructionsTest, PhiMightNotBeFPMathOperator) {
diff --git a/llvm/unittests/IR/ValueTest.cpp b/llvm/unittests/IR/ValueTest.cpp
index 246c2fc7fe4063dafbded88bf3c57164defafa00..33a86d510d45cbd28f7729000353b756e2352316 100644
--- a/llvm/unittests/IR/ValueTest.cpp
+++ b/llvm/unittests/IR/ValueTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 using namespace llvm;
@@ -255,6 +256,8 @@ TEST(ValueTest, getLocalSlotDeath) {
 TEST(ValueTest, replaceUsesOutsideBlock) {
   // Check that Value::replaceUsesOutsideBlock(New, BB) replaces uses outside
   // BB, including dbg.* uses of MetadataAsValue(ValueAsMetadata(this)).
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = false;
   const auto *IR = R"(
     define i32 @f() !dbg !6 {
     entry:
@@ -315,6 +318,7 @@ TEST(ValueTest, replaceUsesOutsideBlock) {
   // These users are outside Entry so should be changed.
   ASSERT_TRUE(ExitDbg->getValue(0) == cast<Value>(B));
   ASSERT_TRUE(Ret->getOperand(0) == cast<Value>(B));
+  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
@@ -359,10 +363,6 @@ TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
   if (!M)
     Err.print("ValueTest", errs());
 
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
-  M->convertToNewDbgValues();
-
   auto GetNext = [](auto *I) { return &*++I->getIterator(); };
 
   Function *F = M->getFunction("f");
@@ -389,7 +389,6 @@ TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
   EXPECT_TRUE(DVR1->getVariableLocationOp(0) == cast<Value>(A));
   // These users are outside Entry so should be changed.
   EXPECT_TRUE(DVR2->getVariableLocationOp(0) == cast<Value>(B));
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 } // end anonymous namespace
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index a913718d0fe06c362e15e6d51653ff0d896402be..15eb59ee00c94c14bcf828bece5ff4026391af11 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -662,4 +662,127 @@ TEST(MemProf, MissingFrameId) {
   ASSERT_TRUE(FrameIdConv.LastUnmappedId.has_value());
   EXPECT_EQ(*FrameIdConv.LastUnmappedId, 3U);
 }
+
+// Verify CallStackRadixTreeBuilder can handle empty inputs.
+TEST(MemProf, RadixTreeBuilderEmpty) {
+  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes;
+  llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
+  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
+      FrameHistogram =
+          llvm::memprof::computeFrameHistogram(MemProfCallStackData);
+  llvm::memprof::CallStackRadixTreeBuilder Builder;
+  Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes,
+                FrameHistogram);
+  ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty());
+  const auto Mappings = Builder.takeCallStackPos();
+  ASSERT_THAT(Mappings, testing::IsEmpty());
+}
+
+// Verify CallStackRadixTreeBuilder can handle one trivial call stack.
+TEST(MemProf, RadixTreeBuilderOne) {
+  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+      {11, 1}, {12, 2}, {13, 3}};
+  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {13, 12, 11};
+  llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
+  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
+      FrameHistogram =
+          llvm::memprof::computeFrameHistogram(MemProfCallStackData);
+  llvm::memprof::CallStackRadixTreeBuilder Builder;
+  Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes,
+                FrameHistogram);
+  EXPECT_THAT(Builder.getRadixArray(), testing::ElementsAreArray({
+                                           3U, // Size of CS1,
+                                           3U, // MemProfFrameIndexes[13]
+                                           2U, // MemProfFrameIndexes[12]
+                                           1U  // MemProfFrameIndexes[11]
+                                       }));
+  const auto Mappings = Builder.takeCallStackPos();
+  ASSERT_THAT(Mappings, SizeIs(1));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS1), 0U)));
+}
+
+// Verify CallStackRadixTreeBuilder can form a link between two call stacks.
+TEST(MemProf, RadixTreeBuilderTwo) {
+  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+      {11, 1}, {12, 2}, {13, 3}};
+  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {12, 11};
+  llvm::SmallVector<llvm::memprof::FrameId> CS2 = {13, 12, 11};
+  llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
+  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
+      FrameHistogram =
+          llvm::memprof::computeFrameHistogram(MemProfCallStackData);
+  llvm::memprof::CallStackRadixTreeBuilder Builder;
+  Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes,
+                FrameHistogram);
+  EXPECT_THAT(Builder.getRadixArray(),
+              testing::ElementsAreArray({
+                  2U,                        // Size of CS1
+                  static_cast<uint32_t>(-3), // Jump 3 steps
+                  3U,                        // Size of CS2
+                  3U,                        // MemProfFrameIndexes[13]
+                  2U,                        // MemProfFrameIndexes[12]
+                  1U                         // MemProfFrameIndexes[11]
+              }));
+  const auto Mappings = Builder.takeCallStackPos();
+  ASSERT_THAT(Mappings, SizeIs(2));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS1), 0U)));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS2), 2U)));
+}
+
+// Verify CallStackRadixTreeBuilder can form a jump to a prefix that itself has
+// another jump to another prefix.
+TEST(MemProf, RadixTreeBuilderSuccessiveJumps) {
+  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+      {11, 1}, {12, 2}, {13, 3}, {14, 4}, {15, 5}, {16, 6}, {17, 7}, {18, 8},
+  };
+  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {14, 13, 12, 11};
+  llvm::SmallVector<llvm::memprof::FrameId> CS2 = {15, 13, 12, 11};
+  llvm::SmallVector<llvm::memprof::FrameId> CS3 = {17, 16, 12, 11};
+  llvm::SmallVector<llvm::memprof::FrameId> CS4 = {18, 16, 12, 11};
+  llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS3), CS3});
+  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS4), CS4});
+  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
+      FrameHistogram =
+          llvm::memprof::computeFrameHistogram(MemProfCallStackData);
+  llvm::memprof::CallStackRadixTreeBuilder Builder;
+  Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes,
+                FrameHistogram);
+  EXPECT_THAT(Builder.getRadixArray(),
+              testing::ElementsAreArray({
+                  4U,                        // Size of CS1
+                  4U,                        // MemProfFrameIndexes[14]
+                  static_cast<uint32_t>(-3), // Jump 3 steps
+                  4U,                        // Size of CS2
+                  5U,                        // MemProfFrameIndexes[15]
+                  3U,                        // MemProfFrameIndexes[13]
+                  static_cast<uint32_t>(-7), // Jump 7 steps
+                  4U,                        // Size of CS3
+                  7U,                        // MemProfFrameIndexes[17]
+                  static_cast<uint32_t>(-3), // Jump 3 steps
+                  4U,                        // Size of CS4
+                  8U,                        // MemProfFrameIndexes[18]
+                  6U,                        // MemProfFrameIndexes[16]
+                  2U,                        // MemProfFrameIndexes[12]
+                  1U                         // MemProfFrameIndexes[11]
+              }));
+  const auto Mappings = Builder.takeCallStackPos();
+  ASSERT_THAT(Mappings, SizeIs(4));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS1), 0U)));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS2), 3U)));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS3), 7U)));
+  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
+                            llvm::memprof::hashCallStack(CS4), 10U)));
+}
 } // namespace
diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp
index 824cf7501fd44f03b63c5615914bbe81aed9cfbf..84882550117d8302f01a68ce60de512ec776154d 100644
--- a/llvm/unittests/Support/KnownBitsTest.cpp
+++ b/llvm/unittests/Support/KnownBitsTest.cpp
@@ -68,8 +68,9 @@ static void testUnaryOpExhaustive(StringRef Name, UnaryBitsFn BitsFn,
         }
       });
 
-      EXPECT_TRUE(!Computed.hasConflict());
-      EXPECT_TRUE(checkResult(Name, Exact, Computed, Known, CheckOptimality));
+      if (!Exact.hasConflict()) {
+        EXPECT_TRUE(checkResult(Name, Exact, Computed, Known, CheckOptimality));
+      }
     });
   }
 }
@@ -95,12 +96,14 @@ static void testBinaryOpExhaustive(StringRef Name, BinaryBitsFn BitsFn,
           });
         });
 
-        EXPECT_TRUE(!Computed.hasConflict());
-        EXPECT_TRUE(checkResult(Name, Exact, Computed, {Known1, Known2},
-                                CheckOptimality));
+        if (!Exact.hasConflict()) {
+          EXPECT_TRUE(checkResult(Name, Exact, Computed, {Known1, Known2},
+                                  CheckOptimality));
+        }
         // In some cases we choose to return zero if the result is always
         // poison.
-        if (RefinePoisonToZero && Exact.hasConflict()) {
+        if (RefinePoisonToZero && Exact.hasConflict() &&
+            !Known1.hasConflict() && !Known2.hasConflict()) {
           EXPECT_TRUE(Computed.isZero());
         }
       });
@@ -135,7 +138,9 @@ TEST(KnownBitsTest, AddCarryExhaustive) {
 
         KnownBits Computed =
             KnownBits::computeForAddCarry(Known1, Known2, KnownCarry);
-        EXPECT_EQ(Exact, Computed);
+        if (!Exact.hasConflict()) {
+          EXPECT_EQ(Exact, Computed);
+        }
       });
     });
   });
@@ -246,7 +251,9 @@ TEST(KnownBitsTest, SubBorrowExhaustive) {
 
         KnownBits Computed =
             KnownBits::computeForSubBorrow(Known1, Known2, KnownBorrow);
-        EXPECT_EQ(Exact, Computed);
+        if (!Exact.hasConflict()) {
+          EXPECT_EQ(Exact, Computed);
+        }
       });
     });
   });
@@ -313,7 +320,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
   testBinaryOpExhaustive(
       "udiv exact",
       [](const KnownBits &Known1, const KnownBits &Known2) {
-        return KnownBits::udiv(Known1, Known2, /*Exact*/ true);
+        return KnownBits::udiv(Known1, Known2, /*Exact=*/true);
       },
       [](const APInt &N1, const APInt &N2) -> std::optional<APInt> {
         if (N2.isZero() || !N1.urem(N2).isZero())
@@ -335,7 +342,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
   testBinaryOpExhaustive(
       "sdiv exact",
       [](const KnownBits &Known1, const KnownBits &Known2) {
-        return KnownBits::sdiv(Known1, Known2, /*Exact*/ true);
+        return KnownBits::sdiv(Known1, Known2, /*Exact=*/true);
       },
       [](const APInt &N1, const APInt &N2) -> std::optional<APInt> {
         if (N2.isZero() || (N1.isMinSignedValue() && N2.isAllOnes()) ||
@@ -394,11 +401,11 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return N1.shl(N2);
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "ushl_ov",
       [](const KnownBits &Known1, const KnownBits &Known2) {
-        return KnownBits::shl(Known1, Known2, /* NUW */ true);
+        return KnownBits::shl(Known1, Known2, /*NUW=*/true);
       },
       [](const APInt &N1, const APInt &N2) -> std::optional<APInt> {
         bool Overflow;
@@ -407,11 +414,11 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return Res;
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "shl nsw",
       [](const KnownBits &Known1, const KnownBits &Known2) {
-        return KnownBits::shl(Known1, Known2, /* NUW */ false, /* NSW */ true);
+        return KnownBits::shl(Known1, Known2, /*NUW=*/false, /*NSW=*/true);
       },
       [](const APInt &N1, const APInt &N2) -> std::optional<APInt> {
         bool Overflow;
@@ -420,11 +427,11 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return Res;
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "shl nuw",
       [](const KnownBits &Known1, const KnownBits &Known2) {
-        return KnownBits::shl(Known1, Known2, /* NUW */ true, /* NSW */ true);
+        return KnownBits::shl(Known1, Known2, /*NUW=*/true, /*NSW=*/true);
       },
       [](const APInt &N1, const APInt &N2) -> std::optional<APInt> {
         bool OverflowUnsigned, OverflowSigned;
@@ -434,7 +441,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return Res;
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
 
   testBinaryOpExhaustive(
       "lshr",
@@ -446,7 +453,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return N1.lshr(N2);
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "lshr exact",
       [](const KnownBits &Known1, const KnownBits &Known2) {
@@ -460,7 +467,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return N1.lshr(N2);
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "ashr",
       [](const KnownBits &Known1, const KnownBits &Known2) {
@@ -471,7 +478,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return N1.ashr(N2);
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "ashr exact",
       [](const KnownBits &Known1, const KnownBits &Known2) {
@@ -485,7 +492,7 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           return std::nullopt;
         return N1.ashr(N2);
       },
-      /*CheckOptimality=*/true, /* RefinePoisonToZero */ true);
+      /*CheckOptimality=*/true, /*RefinePoisonToZero=*/true);
   testBinaryOpExhaustive(
       "mul",
       [](const KnownBits &Known1, const KnownBits &Known2) {
@@ -538,7 +545,7 @@ TEST(KnownBitsTest, UnaryExhaustive) {
   testUnaryOpExhaustive(
       "mul self",
       [](const KnownBits &Known) {
-        return KnownBits::mul(Known, Known, /*SelfMultiply*/ true);
+        return KnownBits::mul(Known, Known, /*SelfMultiply=*/true);
       },
       [](const APInt &N) { return N * N; }, /*CheckOptimality=*/false);
 }
@@ -608,6 +615,9 @@ TEST(KnownBitsTest, ICmpExhaustive) {
       std::optional<bool> KnownSLT = KnownBits::slt(Known1, Known2);
       std::optional<bool> KnownSLE = KnownBits::sle(Known1, Known2);
 
+      if (Known1.hasConflict() || Known2.hasConflict())
+        return;
+
       EXPECT_EQ(AllEQ || NoneEQ, KnownEQ.has_value());
       EXPECT_EQ(AllNE || NoneNE, KnownNE.has_value());
       EXPECT_EQ(AllUGT || NoneUGT, KnownUGT.has_value());
@@ -653,8 +663,10 @@ TEST(KnownBitsTest, GetMinMaxVal) {
       Min = APIntOps::umin(Min, N);
       Max = APIntOps::umax(Max, N);
     });
-    EXPECT_EQ(Min, Known.getMinValue());
-    EXPECT_EQ(Max, Known.getMaxValue());
+    if (!Known.hasConflict()) {
+      EXPECT_EQ(Min, Known.getMinValue());
+      EXPECT_EQ(Max, Known.getMaxValue());
+    }
   });
 }
 
@@ -667,8 +679,10 @@ TEST(KnownBitsTest, GetSignedMinMaxVal) {
       Min = APIntOps::smin(Min, N);
       Max = APIntOps::smax(Max, N);
     });
-    EXPECT_EQ(Min, Known.getSignedMinValue());
-    EXPECT_EQ(Max, Known.getSignedMaxValue());
+    if (!Known.hasConflict()) {
+      EXPECT_EQ(Min, Known.getSignedMinValue());
+      EXPECT_EQ(Max, Known.getSignedMaxValue());
+    }
   });
 }
 
@@ -679,7 +693,9 @@ TEST(KnownBitsTest, CountMaxActiveBits) {
     ForeachNumInKnownBits(Known, [&](const APInt &N) {
       Expected = std::max(Expected, N.getActiveBits());
     });
-    EXPECT_EQ(Expected, Known.countMaxActiveBits());
+    if (!Known.hasConflict()) {
+      EXPECT_EQ(Expected, Known.countMaxActiveBits());
+    }
   });
 }
 
@@ -690,7 +706,9 @@ TEST(KnownBitsTest, CountMaxSignificantBits) {
     ForeachNumInKnownBits(Known, [&](const APInt &N) {
       Expected = std::max(Expected, N.getSignificantBits());
     });
-    EXPECT_EQ(Expected, Known.countMaxSignificantBits());
+    if (!Known.hasConflict()) {
+      EXPECT_EQ(Expected, Known.countMaxSignificantBits());
+    }
   });
 }
 
@@ -698,8 +716,8 @@ TEST(KnownBitsTest, SExtOrTrunc) {
   const unsigned NarrowerSize = 4;
   const unsigned BaseSize = 6;
   const unsigned WiderSize = 8;
-  APInt NegativeFitsNarrower(BaseSize, -4, /*isSigned*/ true);
-  APInt NegativeDoesntFitNarrower(BaseSize, -28, /*isSigned*/ true);
+  APInt NegativeFitsNarrower(BaseSize, -4, /*isSigned=*/true);
+  APInt NegativeDoesntFitNarrower(BaseSize, -28, /*isSigned=*/true);
   APInt PositiveFitsNarrower(BaseSize, 14);
   APInt PositiveDoesntFitNarrower(BaseSize, 36);
   auto InitKnownBits = [&](KnownBits &Res, const APInt &Input) {
@@ -736,8 +754,10 @@ TEST(KnownBitsTest, SExtInReg) {
         CommonZero &= ~Ext;
       });
       KnownBits KnownSExtInReg = Known.sextInReg(FromBits);
-      EXPECT_EQ(CommonOne, KnownSExtInReg.One);
-      EXPECT_EQ(CommonZero, KnownSExtInReg.Zero);
+      if (!Known.hasConflict()) {
+        EXPECT_EQ(CommonOne, KnownSExtInReg.One);
+        EXPECT_EQ(CommonZero, KnownSExtInReg.Zero);
+      }
     });
   }
 }
@@ -752,8 +772,10 @@ TEST(KnownBitsTest, CommonBitsSet) {
           HasCommonBitsSet |= N1.intersects(N2);
         });
       });
-      EXPECT_EQ(!HasCommonBitsSet,
-                KnownBits::haveNoCommonBitsSet(Known1, Known2));
+      if (!Known1.hasConflict() && !Known2.hasConflict()) {
+        EXPECT_EQ(!HasCommonBitsSet,
+                  KnownBits::haveNoCommonBitsSet(Known1, Known2));
+      }
     });
   });
 }
diff --git a/llvm/unittests/Support/KnownBitsTest.h b/llvm/unittests/Support/KnownBitsTest.h
index bc291898814bd1e87995b05c3c9327c4278a2976..807ce31381efbf4e37e9e4a53594a52042c53dc7 100644
--- a/llvm/unittests/Support/KnownBitsTest.h
+++ b/llvm/unittests/Support/KnownBitsTest.h
@@ -26,9 +26,6 @@ template <typename FnTy> void ForeachKnownBits(unsigned Bits, FnTy Fn) {
     for (unsigned One = 0; One < Max; ++One) {
       Known.Zero = Zero;
       Known.One = One;
-      if (Known.hasConflict())
-        continue;
-
       Fn(Known);
     }
   }
@@ -37,13 +34,19 @@ template <typename FnTy> void ForeachKnownBits(unsigned Bits, FnTy Fn) {
 template <typename FnTy>
 void ForeachNumInKnownBits(const KnownBits &Known, FnTy Fn) {
   unsigned Bits = Known.getBitWidth();
-  unsigned Max = 1 << Bits;
-  for (unsigned N = 0; N < Max; ++N) {
-    APInt Num(Bits, N);
-    if ((Num & Known.Zero) != 0 || (~Num & Known.One) != 0)
-      continue;
+  assert(Bits < 32);
+  unsigned Max = 1u << Bits;
+  unsigned Zero = Known.Zero.getZExtValue();
+  unsigned One = Known.One.getZExtValue();
+
+  if (Zero & One) {
+    // Known has a conflict. No values will satisfy it.
+    return;
+  }
 
-    Fn(Num);
+  for (unsigned N = 0; N < Max; ++N) {
+    if ((N & Zero) == 0 && (~N & One) == 0)
+      Fn(APInt(Bits, N));
   }
 }
 
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index e9fd9671ea6ab5f7be07aef67b75f6dad9f10a6a..9e9b4fbcdedd41d0f0cecf3be89710116f5de060 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -1138,6 +1138,11 @@ TEST_F(InMemoryFileSystemTest, DuplicatedFile) {
   ASSERT_FALSE(FS.addFile("/a/b", 0, MemoryBuffer::getMemBuffer("a")));
   ASSERT_TRUE(FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a")));
   ASSERT_FALSE(FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("b")));
+  ASSERT_TRUE(FS.addFile("/b/c/d", 0, MemoryBuffer::getMemBuffer("a")));
+  ASSERT_FALSE(FS.addFile("/b/c", 0, MemoryBuffer::getMemBuffer("a")));
+  ASSERT_TRUE(FS.addFile(
+      "/b/c", 0, MemoryBuffer::getMemBuffer(""), /*User=*/std::nullopt,
+      /*Group=*/std::nullopt, sys::fs::file_type::directory_file));
 }
 
 TEST_F(InMemoryFileSystemTest, DirectoryIteration) {
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index 6aa1d7a087ebf0fbfc9ce50ac06b9e85f41b83db..61921a99e1711ea375f7639c94659e2483381776 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -125,6 +125,9 @@ TEST(getLinuxHostCPUName, AArch64) {
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0xc0\n"
                                               "CPU part        : 0xac5"),
             "ampere1b");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
+                                              "CPU part        : 0x001"),
+            "oryon-1");
 
   // MSM8992/4 weirdness
   StringRef MSM8992ProcCpuInfo = R"(
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 797d7dfbca2003f33b2b582ab5af9a7a1f8880a4..b2e57f2cca13ed1f199320185efbe0868e8f0ebe 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -356,6 +356,11 @@ INSTANTIATE_TEST_SUITE_P(
                                        ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
                                        ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
                                    "8-R"),
+        ARMCPUTestParams<uint64_t>("cortex-r52plus", "armv8-r", "neon-fp-armv8",
+                                   ARM::AEK_NONE | ARM::AEK_CRC | ARM::AEK_MP |
+                                       ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
+                                       ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
+                                   "8-R"),
         ARMCPUTestParams<uint64_t>("sc300", "armv7-m", "none",
                                    ARM::AEK_NONE | ARM::AEK_HWDIVTHUMB, "7-M"),
         ARMCPUTestParams<uint64_t>("cortex-m3", "armv7-m", "none",
@@ -548,7 +553,7 @@ INSTANTIATE_TEST_SUITE_P(
                                    "7-S")),
     ARMCPUTestParams<uint64_t>::PrintToStringParamName);
 
-static constexpr unsigned NumARMCPUArchs = 91;
+static constexpr unsigned NumARMCPUArchs = 92;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector<StringRef, NumARMCPUArchs> List;
@@ -692,6 +697,8 @@ TEST(TargetParserTest, testARMExtension) {
   EXPECT_FALSE(
       testARMExtension("cortex-a75", ARM::ArchKind::INVALID, "fp16fml"));
   EXPECT_FALSE(testARMExtension("cortex-r52", ARM::ArchKind::INVALID, "ras"));
+  EXPECT_FALSE(
+      testARMExtension("cortex-r52plus", ARM::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testARMExtension("iwmmxt", ARM::ArchKind::INVALID, "crc"));
   EXPECT_FALSE(testARMExtension("xscale", ARM::ArchKind::INVALID, "crc"));
   EXPECT_FALSE(testARMExtension("swift", ARM::ArchKind::INVALID, "crc"));
@@ -1815,11 +1822,23 @@ INSTANTIATE_TEST_SUITE_P(
                 {AArch64::AEK_CRC, AArch64::AEK_AES, AArch64::AEK_SHA2,
                  AArch64::AEK_FP, AArch64::AEK_SIMD, AArch64::AEK_FP16,
                  AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_RDM}),
-            "8.2-A")),
+            "8.2-A"),
+        ARMCPUTestParams<AArch64::ExtensionBitset>(
+            "oryon-1", "armv8.6-a", "crypto-neon-fp-armv8",
+            (AArch64::ExtensionBitset(
+                {AArch64::AEK_CRC,     AArch64::AEK_FP,      AArch64::AEK_PAUTH,
+                 AArch64::AEK_FCMA,    AArch64::AEK_JSCVT,   AArch64::AEK_SIMD,
+                 AArch64::AEK_RAS,     AArch64::AEK_LSE,     AArch64::AEK_RDM,
+                 AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_SM4,
+                 AArch64::AEK_SHA3,    AArch64::AEK_BF16,    AArch64::AEK_SHA2,
+                 AArch64::AEK_AES,     AArch64::AEK_I8MM,    AArch64::AEK_RAND,
+                 AArch64::AEK_PROFILE, AArch64::AEK_CRYPTO})),
+            "8.6-A")),
+
     ARMCPUTestParams<AArch64::ExtensionBitset>::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 76;
+static constexpr unsigned NumAArch64CPUArchs = 77;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector<StringRef, NumAArch64CPUArchs> List;
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 5e302d9c0a0d3e9d4de45f3cb557bfd9195c9277..1d0d56a2099ceb2d824cffb17005b2deccaff89b 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -844,8 +844,9 @@ TEST(CloneFunction, CloneFunctionWithInlinedSubprograms) {
   EXPECT_FALSE(verifyModule(*ImplModule, &errs()));
 
   // Check that DILexicalBlock of inlined function was not cloned.
-  auto DbgDeclareI = Func->begin()->begin();
-  auto ClonedDbgDeclareI = ClonedFunc->begin()->begin();
+  auto DbgDeclareI = Func->begin()->begin()->getDbgRecordRange().begin();
+  auto ClonedDbgDeclareI =
+      ClonedFunc->begin()->begin()->getDbgRecordRange().begin();
   const DebugLoc &DbgLoc = DbgDeclareI->getDebugLoc();
   const DebugLoc &ClonedDbgLoc = ClonedDbgDeclareI->getDebugLoc();
   EXPECT_NE(DbgLoc.get(), ClonedDbgLoc.get());
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 9b1176765c17f12b824bb1437e4c9dff6b75d5a9..316d59a9d229695bea5feb0589961598babd0e11 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/PostDominators.h"
@@ -26,6 +27,27 @@
 
 using namespace llvm;
 
+extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
+extern cl::opt<cl::boolOrDefault> PreserveInputDbgFormat;
+extern bool WriteNewDbgInfoFormatToBitcode;
+extern cl::opt<bool> WriteNewDbgInfoFormat;
+
+// Backup all of the existing settings that may be modified when
+// PreserveInputDbgFormat=true, so that when the test is finished we return them
+// (and the "preserve" setting) to their original values.
+static auto SaveDbgInfoFormat() {
+  return make_scope_exit(
+      [OldPreserveInputDbgFormat = PreserveInputDbgFormat.getValue(),
+       OldUseNewDbgInfoFormat = UseNewDbgInfoFormat.getValue(),
+       OldWriteNewDbgInfoFormatToBitcode = WriteNewDbgInfoFormatToBitcode,
+       OldWriteNewDbgInfoFormat = WriteNewDbgInfoFormat.getValue()] {
+        PreserveInputDbgFormat = OldPreserveInputDbgFormat;
+        UseNewDbgInfoFormat = OldUseNewDbgInfoFormat;
+        WriteNewDbgInfoFormatToBitcode = OldWriteNewDbgInfoFormatToBitcode;
+        WriteNewDbgInfoFormat = OldWriteNewDbgInfoFormat;
+      });
+}
+
 TEST(Local, RecursivelyDeleteDeadPHINodes) {
   LLVMContext C;
 
@@ -116,7 +138,6 @@ static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
 
 TEST(Local, ReplaceDbgDeclare) {
   LLVMContext C;
-
   // Original C source to get debug info for a local variable:
   // void f() { int x; }
   std::unique_ptr<Module> M = parseIR(C,
@@ -124,11 +145,11 @@ TEST(Local, ReplaceDbgDeclare) {
       define void @f() !dbg !8 {
       entry:
         %x = alloca i32, align 4
-        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
-        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
+          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
+          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
         ret void, !dbg !14
       }
-      declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -151,20 +172,18 @@ TEST(Local, ReplaceDbgDeclare) {
   Instruction *Inst = &F->front().front();
   auto *AI = dyn_cast<AllocaInst>(Inst);
   ASSERT_TRUE(AI);
-  Inst = Inst->getNextNode()->getNextNode();
-  ASSERT_TRUE(Inst);
-  auto *DII = dyn_cast<DbgDeclareInst>(Inst);
-  ASSERT_TRUE(DII);
+
   Value *NewBase = Constant::getNullValue(PointerType::getUnqual(C));
   DIBuilder DIB(*M);
   replaceDbgDeclare(AI, NewBase, DIB, DIExpression::ApplyOffset, 0);
 
-  // There should be exactly two dbg.declares.
-  int Declares = 0;
-  for (const Instruction &I : F->front())
-    if (isa<DbgDeclareInst>(I))
-      Declares++;
-  EXPECT_EQ(2, Declares);
+  // There should be exactly two dbg.declares, attached to the terminator.
+  Inst = F->front().getTerminator();
+  ASSERT_TRUE(Inst);
+  EXPECT_TRUE(Inst->hasDbgRecords());
+  EXPECT_EQ(range_size(Inst->getDbgRecordRange()), 2u);
+  for (DbgVariableRecord &DVR : filterDbgVars(Inst->getDbgRecordRange()))
+    EXPECT_EQ(DVR.getAddress(), NewBase);
 }
 
 /// Build the dominator tree for the function and run the Test.
@@ -499,11 +518,10 @@ struct SalvageDebugInfoTest : ::testing::Test {
       entry:
         %x = add i32 0, 1
         %y = add i32 %x, 2
-        call void @llvm.dbg.value(metadata i32 %x, metadata !11, metadata !DIExpression()), !dbg !13
-        call void @llvm.dbg.value(metadata i32 %y, metadata !11, metadata !DIExpression()), !dbg !13
+          #dbg_value(i32 %x, !11, !DIExpression(), !13)
+          #dbg_value(i32 %y, !11, !DIExpression(), !13)
         ret void, !dbg !14
       }
-      declare void @llvm.dbg.value(metadata, metadata, metadata)
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -526,48 +544,47 @@ struct SalvageDebugInfoTest : ::testing::Test {
     ASSERT_TRUE(F);
   }
 
-  bool doesDebugValueDescribeX(const DbgValueInst &DI) {
-    if (DI.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeX(const DbgVariableRecord &DVR) {
+    if (DVR.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DI.getValue(0));
+    const auto &CI = *cast<ConstantInt>(DVR.getValue(0));
     if (CI.isZero())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 1, dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DI.getExpression()->getElements().empty();
+      return DVR.getExpression()->getElements().empty();
     return false;
   }
 
-  bool doesDebugValueDescribeY(const DbgValueInst &DI) {
-    if (DI.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeY(const DbgVariableRecord &DVR) {
+    if (DVR.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DI.getVariableLocationOp(0));
+    const auto &CI = *cast<ConstantInt>(DVR.getVariableLocationOp(0));
     if (CI.isZero())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 3, dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 2, dwarf::DW_OP_stack_value});
     return false;
   }
 
   void verifyDebugValuesAreSalvaged() {
+    // The function should only contain debug values and a terminator.
+    EXPECT_EQ(F->size(), 1u);
+    EXPECT_TRUE(F->begin()->begin()->isTerminator());
+
     // Check that the debug values for %x and %y are preserved.
     bool FoundX = false;
     bool FoundY = false;
-    for (const Instruction &I : F->front()) {
-      auto DI = dyn_cast<DbgValueInst>(&I);
-      if (!DI) {
-        // The function should only contain debug values and a terminator.
-        ASSERT_TRUE(I.isTerminator());
-        continue;
-      }
-      EXPECT_EQ(DI->getVariable()->getName(), "x");
-      FoundX |= doesDebugValueDescribeX(*DI);
-      FoundY |= doesDebugValueDescribeY(*DI);
+    for (DbgVariableRecord &DVR :
+         filterDbgVars(F->begin()->begin()->getDbgRecordRange())) {
+      EXPECT_EQ(DVR.getVariable()->getName(), "x");
+      FoundX |= doesDebugValueDescribeX(DVR);
+      FoundY |= doesDebugValueDescribeY(DVR);
     }
-    ASSERT_TRUE(FoundX);
-    ASSERT_TRUE(FoundY);
+    EXPECT_TRUE(FoundX);
+    EXPECT_TRUE(FoundY);
   }
 };
 
@@ -590,6 +607,12 @@ TEST_F(SalvageDebugInfoTest, RecursiveBlockSimplification) {
 
 TEST(Local, wouldInstructionBeTriviallyDead) {
   LLVMContext Ctx;
+  // FIXME: PreserveInputDbgFormat is set to true because this test has
+  // been written to expect debug intrinsics rather than debug records.
+  // TODO: This test doesn't have a DbgRecord equivalent form so delete
+  // it when debug intrinsics are removed.
+  auto SettingGuard = SaveDbgInfoFormat();
+  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
   std::unique_ptr<Module> M = parseIR(Ctx,
                                       R"(
     define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
@@ -683,12 +706,10 @@ TEST(Local, FindDbgUsers) {
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
+      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
     ret void
   }
 
-  declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
-
   !llvm.dbg.cu = !{!0}
   !llvm.module.flags = !{!2, !3, !9}
   !llvm.ident = !{!10}
@@ -715,9 +736,13 @@ TEST(Local, FindDbgUsers) {
   verifyModule(*M, &errs(), &BrokenDebugInfo);
   ASSERT_FALSE(BrokenDebugInfo);
 
+  // Convert to debug intrinsics as we want to test findDbgUsers and
+  // findDbgValue's debug-intrinsic-finding code here.
+  // TODO: Remove this test when debug intrinsics are removed.
+  M->convertFromNewDbgValues();
+
   Function &Fun = *cast<Function>(M->getNamedValue("fun"));
   Value *Arg = Fun.getArg(0);
-
   SmallVector<DbgVariableIntrinsic *> Users;
   // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
   // only 1 pointer to it rather than 2.
@@ -738,7 +763,7 @@ TEST(Local, FindDbgRecords) {
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
+      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
     ret void
   }
 
@@ -767,9 +792,6 @@ TEST(Local, FindDbgRecords) {
   bool BrokenDebugInfo = true;
   verifyModule(*M, &errs(), &BrokenDebugInfo);
   ASSERT_FALSE(BrokenDebugInfo);
-  bool NewDbgInfoFormat = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
-  M->convertToNewDbgValues();
 
   Function &Fun = *cast<Function>(M->getNamedValue("fun"));
   Value *Arg = Fun.getArg(0);
@@ -789,12 +811,10 @@ TEST(Local, FindDbgRecords) {
   findDbgValues(Vals, Arg, &Records);
   EXPECT_EQ(Vals.size(), 0u);
   EXPECT_EQ(Records.size(), 1u);
-  UseNewDbgInfoFormat = NewDbgInfoFormat;
 }
 
 TEST(Local, ReplaceAllDbgUsesWith) {
   using namespace llvm::dwarf;
-
   LLVMContext Ctx;
 
   // Note: The datalayout simulates Darwin/x86_64.
@@ -807,39 +827,36 @@ TEST(Local, ReplaceAllDbgUsesWith) {
     define void @f() !dbg !6 {
     entry:
       %a = add i32 0, 1, !dbg !15
-      call void @llvm.dbg.value(metadata i32 %a, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %a, !9, !DIExpression(), !15)
       %b = add i64 0, 1, !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression()), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
 
-      %c = inttoptr i64 0 to i64*, !dbg !17
-      call void @llvm.dbg.declare(metadata i64* %c, metadata !13, metadata !DIExpression()), !dbg !17
+        #dbg_value(i64 %b, !11, !DIExpression(), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8), !16)
+      %c = inttoptr i64 0 to ptr, !dbg !17
 
-      %d = inttoptr i64 0 to i32*, !dbg !18
-      call void @llvm.dbg.declare(metadata i32* %d, metadata !20, metadata !DIExpression()), !dbg !18
+        #dbg_declare(ptr %c, !13, !DIExpression(), !17)
+      %d = inttoptr i64 0 to ptr, !dbg !18
 
+        #dbg_declare(ptr %d,  !20,  !DIExpression(), !18)
       %e = add <2 x i16> zeroinitializer, zeroinitializer
-      call void @llvm.dbg.value(metadata <2 x i16> %e, metadata !14, metadata !DIExpression()), !dbg !18
 
+        #dbg_value(<2 x i16> %e, !14, !DIExpression(), !18)
       %f = call i32 @escape(i32 0)
-      call void @llvm.dbg.value(metadata i32 %f, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %f, !9, !DIExpression(), !15)
       %barrier = call i32 @escape(i32 0)
 
       %g = call i32 @escape(i32 %f)
-      call void @llvm.dbg.value(metadata i32 %g, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %g, !9, !DIExpression(), !15)
       ret void, !dbg !19
     }
 
-    declare void @llvm.dbg.declare(metadata, metadata, metadata)
-    declare void @llvm.dbg.value(metadata, metadata, metadata)
-
     !llvm.dbg.cu = !{!0}
     !llvm.module.flags = !{!5}
 
@@ -894,38 +911,47 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   EXPECT_TRUE(replaceAllDbgUsesWith(D, C, C, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> CDbgVals;
-  findDbgUsers(CDbgVals, &C);
-  EXPECT_EQ(2U, CDbgVals.size());
-  EXPECT_TRUE(all_of(CDbgVals, [](DbgVariableIntrinsic *DII) {
-    return isa<DbgDeclareInst>(DII);
-  }));
+  SmallVector<DbgVariableRecord *, 2> CDbgRecords;
+  findDbgUsers(CDbgVals, &C, &CDbgRecords);
+  EXPECT_EQ(0U, CDbgVals.size());
+  EXPECT_EQ(2U, CDbgRecords.size());
+  EXPECT_TRUE(all_of(
+      CDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
 
   EXPECT_TRUE(replaceAllDbgUsesWith(C, D, D, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> DDbgVals;
-  findDbgUsers(DDbgVals, &D);
-  EXPECT_EQ(2U, DDbgVals.size());
-  EXPECT_TRUE(all_of(DDbgVals, [](DbgVariableIntrinsic *DII) {
-    return isa<DbgDeclareInst>(DII);
-  }));
+  SmallVector<DbgVariableRecord *, 2> DDbgRecords;
+  findDbgUsers(DDbgVals, &D, &DDbgRecords);
+  EXPECT_EQ(0U, DDbgVals.size());
+  EXPECT_EQ(2U, DDbgRecords.size());
+  EXPECT_TRUE(all_of(
+      DDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
 
   // Introduce a use-before-def. Check that the dbg.value for %a is salvaged.
   EXPECT_TRUE(replaceAllDbgUsesWith(A, F_, F_, DT));
 
-  auto *ADbgVal = cast<DbgValueInst>(A.getNextNode());
-  EXPECT_EQ(ADbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_EQ(ConstantInt::get(A.getType(), 0), ADbgVal->getVariableLocationOp(0));
+  EXPECT_FALSE(A.hasDbgRecords());
+  EXPECT_TRUE(B.hasDbgRecords());
+  DbgVariableRecord *BDbgVal =
+      cast<DbgVariableRecord>(&*B.getDbgRecordRange().begin());
+  EXPECT_EQ(BDbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_EQ(ConstantInt::get(A.getType(), 0),
+            BDbgVal->getVariableLocationOp(0));
 
   // Introduce a use-before-def. Check that the dbg.values for %f become undef.
   EXPECT_TRUE(replaceAllDbgUsesWith(F_, G, G, DT));
 
-  auto *FDbgVal = cast<DbgValueInst>(F_.getNextNode());
-  EXPECT_EQ(FDbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_TRUE(FDbgVal->isKillLocation());
+  DbgVariableRecord *BarrierDbgVal =
+      cast<DbgVariableRecord>(&*Barrier.getDbgRecordRange().begin());
+  EXPECT_EQ(BarrierDbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_TRUE(BarrierDbgVal->isKillLocation());
 
-  SmallVector<DbgValueInst *, 1> FDbgVals;
-  findDbgValues(FDbgVals, &F_);
-  EXPECT_EQ(0U, FDbgVals.size());
+  SmallVector<DbgValueInst *, 1> BarrierDbgVals;
+  SmallVector<DbgVariableRecord *, 8> BarrierDbgRecs;
+  findDbgValues(BarrierDbgVals, &F_, &BarrierDbgRecs);
+  EXPECT_EQ(0U, BarrierDbgVals.size());
+  EXPECT_EQ(0U, BarrierDbgRecs.size());
 
   // Simulate i32 -> i64 conversion to test sign-extension. Here are some
   // interesting cases to handle:
@@ -935,13 +961,15 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   //  4-6) like (1-3), but with a fragment
   EXPECT_TRUE(replaceAllDbgUsesWith(B, A, A, DT));
 
-  SmallVector<DbgValueInst *, 8> ADbgVals;
-  findDbgValues(ADbgVals, &A);
-  EXPECT_EQ(6U, ADbgVals.size());
+  SmallVector<DbgValueInst *, 8> BDbgVals;
+  SmallVector<DbgVariableRecord *, 8> BDbgRecs;
+  findDbgValues(BDbgVals, &A, &BDbgRecs);
+  EXPECT_EQ(0U, BDbgVals.size());
+  EXPECT_EQ(6U, BDbgRecs.size());
 
   // Check that %a has a dbg.value with a DIExpression matching \p Ops.
   auto hasADbgVal = [&](ArrayRef<uint64_t> Ops) {
-    return any_of(ADbgVals, [&](DbgValueInst *DVI) {
+    return any_of(BDbgRecs, [&](DbgVariableRecord *DVI) {
       assert(DVI->getVariable()->getName() == "2");
       return DVI->getExpression()->getElements() == Ops;
     });
@@ -1344,6 +1372,11 @@ TEST(Local, ExpressionForConstant) {
 
 TEST(Local, ReplaceDbgVariableRecord) {
   LLVMContext C;
+  // FIXME: PreserveInputDbgFormat is set to true because this test has
+  // been written to expect debug intrinsics rather than debug records; use the
+  // intrinsic format until we update the test checks.
+  auto SettingGuard = SaveDbgInfoFormat();
+  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
 
   // Test that RAUW also replaces the operands of DbgVariableRecord objects,
   // i.e. non-instruction stored debugging information.
diff --git a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
index 210dd127850951e559e63391a21fddc9ce965cdc..e88df022be7842f8cac05b60f0b5d4d8ce80a2ca 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
@@ -29,6 +29,7 @@ static_library("Core") {
     "DynoStats.cpp",
     "Exceptions.cpp",
     "FunctionLayout.cpp",
+    "GDBIndex.cpp",
     "HashUtilities.cpp",
     "JumpTable.cpp",
     "MCPlusBuilder.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
index 36957f502c32317228d921a9cbb6f8352e258044..0dc5efc981c8729c0c9b50df97a8a807a868d1fe 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
@@ -55,5 +55,6 @@ static_library("misc") {
     "UnusedParametersCheck.cpp",
     "UnusedUsingDeclsCheck.cpp",
     "UseAnonymousNamespaceCheck.cpp",
+    "UseInternalLinkageCheck.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
index 0cf99256c9bdb6fc89edb374c02d2d055754ebad..d3a3ee755808bfddb30270721d6892f8f1c2a82b 100644
--- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -110,6 +110,7 @@ static_library("AST") {
     "Interp/InterpShared.cpp",
     "Interp/InterpStack.cpp",
     "Interp/InterpState.cpp",
+    "Interp/MemberPointer.cpp",
     "Interp/Pointer.cpp",
     "Interp/PrimType.cpp",
     "Interp/Program.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index 3ae50b214eb173087eb82368d6ad9d13203f085d..f12aaa7dd4fad64c2fa2b2e2d5485409869b6528 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -83,6 +83,7 @@ static_library("Checkers") {
     "MoveChecker.cpp",
     "NSAutoreleasePoolChecker.cpp",
     "NSErrorChecker.cpp",
+    "NoOwnershipChangeVisitor.cpp",
     "NoReturnFunctionChecker.cpp",
     "NonNullParamChecker.cpp",
     "NonnullGlobalConstantsChecker.cpp",
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 2575dbf3cd0dd0edf7c03c77d7fc12163690e0fe..1ec94a419f56c8d443e487a1a69e2967d3af7bcd 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -201,6 +201,7 @@ group("test") {
     "//llvm/tools/llvm-readobj:symlinks",
     "//llvm/tools/llvm-readtapi:symlinks",
     "//llvm/tools/llvm-symbolizer:symlinks",
+    "//llvm/tools/obj2yaml",
     "//llvm/tools/opt",
     "//llvm/tools/yaml2obj",
     "//llvm/utils/FileCheck",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 0d4725716191d28c69a9be0532cd4bfa91abfff9..ee5905ae5a7d283b8d5cd2eb7e3cbfe90786aefd 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -190,7 +190,10 @@ static_library("Support") {
   if (current_os == "linux" || current_os == "android") {
     libs += [ "dl" ]
   } else if (current_os == "win") {
-    libs = [ "ws2_32.lib" ]
+    libs = [
+      "ntdll.lib",
+      "ws2_32.lib",
+    ]
 
     # Delay load shell32.dll if possible to speed up process startup.
     libs += [ "delayimp.lib" ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
index 879b7f0960a90ee88f5830a92b768b6651a52588..2ffe83da90eed79f62c3bb2b8310124ec3b3dda8 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
@@ -103,6 +103,7 @@ static_library("LLVMAArch64CodeGen") {
     "//llvm/lib/Transforms/CFGuard",
     "//llvm/lib/Transforms/Scalar",
     "//llvm/lib/Transforms/Utils",
+    "//llvm/lib/Transforms/Vectorize",
   ]
   include_dirs = [ "." ]
   sources = [
@@ -130,7 +131,6 @@ static_library("LLVMAArch64CodeGen") {
     "AArch64ISelLowering.cpp",
     "AArch64InstrInfo.cpp",
     "AArch64LoadStoreOptimizer.cpp",
-    "AArch64LoopIdiomTransform.cpp",
     "AArch64LowerHomogeneousPrologEpilog.cpp",
     "AArch64MCInstLower.cpp",
     "AArch64MIPeepholeOpt.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
index 822c2ec8afc65303295d10c84adc1eb1b28a7fa0..248e47a9584cdf20bba17f34bdf1974b0ab84c9d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
@@ -34,6 +34,7 @@ static_library("LLVMLoongArchCodeGen") {
   include_dirs = [ "." ]
   sources = [
     "LoongArchAsmPrinter.cpp",
+    "LoongArchDeadRegisterDefinitions.cpp",
     "LoongArchExpandAtomicPseudoInsts.cpp",
     "LoongArchExpandPseudoInsts.cpp",
     "LoongArchFrameLowering.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
index 0d134c7bdffb704df8de8f182e98186685563626..bcf2ea7510568d85b09dcecb344ae91f20b4c2bc 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
@@ -33,6 +33,7 @@ static_library("IPO") {
     "DeadArgumentElimination.cpp",
     "ElimAvailExtern.cpp",
     "EmbedBitcodePass.cpp",
+    "ExpandVariadics.cpp",
     "ExtractGV.cpp",
     "ForceFunctionAttrs.cpp",
     "FunctionAttrs.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 044b781e7c31fbda0f9b5b3b0e59b3b450638205..92337a547eaa3f3cd9f7d1f11b207aa3ba5af33d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -8,6 +8,7 @@ static_library("Vectorize") {
   ]
   sources = [
     "LoadStoreVectorizer.cpp",
+    "LoopIdiomVectorize.cpp",
     "LoopVectorizationLegality.cpp",
     "LoopVectorize.cpp",
     "SLPVectorizer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
index 711e4e3b43151118d065a9fe48d0922dd4a7766f..ddf0e55b4f00f99ba78148bfb4b1da14be4adb37 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
@@ -46,7 +46,7 @@ write_cmake_config("BuildVariables.inc") {
   if (host_os == "win") {
     # uuid.lib for FOLDERID_Profile in lib/Support/Windows/Path.inc.
     # advapi32.lib for CryptAcquireContextW in lib/Support/Windows/Path.inc.
-    system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib"
+    system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib ntdll.lib"
   } else {
     system_libs += "-lm"
     if (host_os == "linux") {
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 1d4babc99984bf521c6a2dea7961c896aca52b04..afb7f078072fc49168e80b5cb39dcb9b6c9baf0e 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -588,7 +588,10 @@ class LLVMConfig(object):
             if getattr(self.config, pp, None)
         ]
 
-        self.with_environment("LD_LIBRARY_PATH", lib_paths, append_path=True)
+        if platform.system() == "AIX":
+            self.with_environment("LIBPATH", lib_paths, append_path=True)
+        else:
+            self.with_environment("LD_LIBRARY_PATH", lib_paths, append_path=True)
 
         shl = getattr(self.config, "llvm_shlib_dir", None)
         pext = getattr(self.config, "llvm_plugin_ext", None)
diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h
index e754318d6685636c4fa3586a3cd57e9990d2d708..902b45444d6c4ef8f905ad690dfc1ea5002a4a30 100644
--- a/mlir/include/mlir-c/Dialect/LLVM.h
+++ b/mlir/include/mlir-c/Dialect/LLVM.h
@@ -251,6 +251,12 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIDerivedTypeAttrGet(
     MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits,
     uint64_t offsetInBits, int64_t dwarfAddressSpace, MlirAttribute extraData);
 
+MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIStringTypeAttrGet(
+    MlirContext ctx, unsigned int tag, MlirAttribute name, uint64_t sizeInBits,
+    uint32_t alignInBits, MlirAttribute stringLength,
+    MlirAttribute stringLengthExp, MlirAttribute stringLocationExp,
+    MlirLLVMTypeEncoding encoding);
+
 /// Constant to represent std::nullopt for dwarfAddressSpace to omit the field.
 #define MLIR_CAPI_DWARF_ADDRESS_SPACE_NULL -1
 
diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h
index 32abacf353133eba903ddc52256ee734b8749669..e3d69b7708b3d65f691ad07665048ae96905a9cf 100644
--- a/mlir/include/mlir-c/IR.h
+++ b/mlir/include/mlir-c/IR.h
@@ -858,6 +858,9 @@ MLIR_CAPI_EXPORTED MlirValue mlirBlockAddArgument(MlirBlock block,
                                                   MlirType type,
                                                   MlirLocation loc);
 
+/// Erase the argument at 'index' and remove it from the argument list.
+MLIR_CAPI_EXPORTED void mlirBlockEraseArgument(MlirBlock block, unsigned index);
+
 /// Inserts an argument of the specified type at a specified index to the block.
 /// Returns the newly added argument.
 MLIR_CAPI_EXPORTED MlirValue mlirBlockInsertArgument(MlirBlock block,
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 3640055ea8da8f7c12adb8be315b2e77f598b4a8..dbec741cf1b1f35e88159d6be7687f5f99f56da9 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -118,8 +118,8 @@ def AffineForOp : Affine_Op<"for",
     [AttrSizedOperandSegments, AutomaticAllocationScope,
      ImplicitAffineTerminator, ConditionallySpeculatable,
      RecursiveMemoryEffects, DeclareOpInterfaceMethods<LoopLikeOpInterface,
-     ["getSingleInductionVar", "getSingleLowerBound", "getSingleStep",
-      "getSingleUpperBound", "getYieldedValuesMutable",
+     ["getLoopInductionVars", "getLoopLowerBounds", "getLoopSteps",
+      "getLoopUpperBounds", "getYieldedValuesMutable",
       "replaceWithAdditionalYields"]>,
      DeclareOpInterfaceMethods<RegionBranchOpInterface,
      ["getEntrySuccessorOperands"]>]> {
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
index 156744ba57e7b2786c0042cb9766ec31d83366f7..167e5b787d1afaf4cb35b53317b4ca8a526cc74d 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
@@ -27,7 +27,8 @@ namespace arm_sme {
 /// Pass to enable Armv9 Streaming SVE mode.
 std::unique_ptr<Pass> createEnableArmStreamingPass(
     const ArmStreamingMode = ArmStreamingMode::Streaming,
-    const ArmZaMode = ArmZaMode::Disabled, bool onlyIfRequiredByOps = false);
+    const ArmZaMode = ArmZaMode::Disabled, bool ifRequiredByOps = false,
+    bool ifContainsScalableVectors = false);
 
 /// Pass that fuses 'arm_sme.outerproduct' ops into 2-way or 4-way widening
 /// variants.
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
index 869a031d6cae89a2e778a80915d53584b3042fb3..c1f016d9ce1f16f5216f7b5796e4ad40ada28732 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
@@ -116,10 +116,14 @@ def EnableArmStreaming
                             "not be used for input and/or output and the "
                             "function must return with ZA unchanged")
            )}]>,
-    Option<"onlyIfRequiredByOps", "only-if-required-by-ops", "bool",
+    Option<"ifRequiredByOps", "if-required-by-ops", "bool",
            /*default=*/"false",
-           "Only apply the selected streaming/ZA modes if the function "
-           " contains ops that require them.">
+           "Only apply the selected streaming/ZA modes if the function contains"
+           " ops that implement the ArmSMETileOpInterface.">,
+    Option<"ifContainsScalableVectors", "if-contains-scalable-vectors",
+           "bool", /*default=*/"false",
+           "Only apply the selected streaming/ZA modes if the function contains"
+           " operations that use scalable vector types.">
   ];
   let dependentDialects = ["func::FuncDialect"];
 }
diff --git a/mlir/include/mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td b/mlir/include/mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td
index 160730480394272ddd5f416a4e9974ed196db22f..582ab91311974b29aaadeee9ecf0ec2bf5672632 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td
@@ -37,17 +37,18 @@ def GPUTargetAttrInterface : AttrInterface<"TargetAttrInterface"> {
         is meant to be used for passing additional options that are not in the
         attribute.
       }],
-      "std::optional<SmallVector<char, 0>>", "serializeToObject",
-      (ins "Operation*":$module, "const gpu::TargetOptions&":$options)>,
+      "std::optional<::mlir::SmallVector<char, 0>>", "serializeToObject",
+      (ins "::mlir::Operation*":$module,
+           "const ::mlir::gpu::TargetOptions&":$options)>,
     InterfaceMethod<[{
         Creates a GPU object attribute from a binary string.
 
         The `object` parameter is a binary string. The `options` parameter is
         meant to be used for passing additional options that are not in the
         attribute.
-      }], "Attribute", "createObject",
-        (ins "const SmallVector<char, 0>&":$object,
-             "const gpu::TargetOptions&":$options)>
+      }], "::mlir::Attribute", "createObject",
+        (ins "const ::llvm::SmallVector<char, 0>&":$object,
+             "const ::mlir::gpu::TargetOptions&":$options)>
   ];
 }
 
@@ -112,9 +113,10 @@ def OffloadingLLVMTranslationAttrInterface :
         The first argument has to be a GPU binary operation.
         If the function fails at any point, it must return `failure`.
       }],
-      "LogicalResult", "embedBinary",
-      (ins "Operation*":$binaryOp, "llvm::IRBuilderBase&":$hostBuilder,
-           "LLVM::ModuleTranslation&":$hostModuleTranslation)
+      "::mlir::LogicalResult", "embedBinary",
+      (ins "::mlir::Operation*":$binaryOp,
+           "::llvm::IRBuilderBase&":$hostBuilder,
+           "::mlir::LLVM::ModuleTranslation&":$hostModuleTranslation)
     >,
     InterfaceMethod<[{
         Translates a `gpu.launch_func` op into a sequence of LLVM IR
@@ -128,10 +130,10 @@ def OffloadingLLVMTranslationAttrInterface :
         respectively. If the function fails at any point, it must return
         `failure`.
       }],
-      "LogicalResult", "launchKernel",
-      (ins "Operation*":$launchFunc, "Operation*":$binaryOp,
-           "llvm::IRBuilderBase&":$hostBuilder,
-           "LLVM::ModuleTranslation&":$hostModuleTranslation)
+      "::mlir::LogicalResult", "launchKernel",
+      (ins "::mlir::Operation*":$launchFunc, "::mlir::Operation*":$binaryOp,
+           "::llvm::IRBuilderBase&":$hostBuilder,
+           "::mlir::LLVM::ModuleTranslation&":$hostModuleTranslation)
     >
   ];
 }
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 10719aae5c8b4666e32ab88baa6865fcb7f5e1c9..eb81b6469746f4a77e6d730755cc0a5a4a04d2f7 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1241,7 +1241,7 @@ def GPU_BarrierOp : GPU_Op<"barrier"> {
 def GPU_GPUModuleOp : GPU_Op<"module", [
       DataLayoutOpInterface, HasDefaultDLTIDataLayout, IsolatedFromAbove,
       SymbolTable, Symbol, SingleBlockImplicitTerminator<"ModuleEndOp">
-    ]>, Arguments<(ins
+    ]>, Arguments<(ins SymbolNameAttr:$sym_name,
           OptionalAttr<GPUNonEmptyTargetArrayAttr>:$targets,
           OptionalAttr<OffloadingTranslationAttr>:$offloadingHandler)> {
   let summary = "A top level compilation unit containing code to be run on a GPU.";
diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
index 123ce36cb0a79513443c1896c652e3739550737c..852490cf7428f8546c3451d9c66ba92df3dac4a8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
@@ -33,36 +33,36 @@ class LLVMFuncOp;
 /// external C function calls. The list of functions provided here must be
 /// implemented separately (e.g. as part of a support runtime library or as part
 /// of the libc).
-LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(ModuleOp moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(Operation *moduleOp);
 /// Declares a function to print a C-string.
 /// If a custom runtime function is defined via `runtimeFunctionName`, it must
 /// have the signature void(char const*). The default function is `printString`.
 LLVM::LLVMFuncOp
-lookupOrCreatePrintStringFn(ModuleOp moduleOp,
+lookupOrCreatePrintStringFn(Operation *moduleOp,
                             std::optional<StringRef> runtimeFunctionName = {});
-LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreateMallocFn(Operation *moduleOp, Type indexType);
+LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(Operation *moduleOp,
                                               Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp lookupOrCreateFreeFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(Operation *moduleOp,
                                               Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp,
                                                      Type indexType);
-LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp);
-LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType,
+LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(Operation *moduleOp);
+LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
                                             Type unrankedDescriptorType);
 
 /// Create a FuncOp with signature `resultType`(`paramTypes`)` and name `name`.
-LLVM::LLVMFuncOp lookupOrCreateFn(ModuleOp moduleOp, StringRef name,
+LLVM::LLVMFuncOp lookupOrCreateFn(Operation *moduleOp, StringRef name,
                                   ArrayRef<Type> paramTypes = {},
                                   Type resultType = {}, bool isVarArg = false);
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index ec58cf368b593ae0a75c01d7640261c4ccac8229..b05366d2a635df13292a5b9efe7fa1c0c3b09f4a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -676,6 +676,25 @@ def LLVM_DILabelAttr : LLVM_Attr<"DILabel", "di_label",
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// DIStringTypeAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_DIStringTypeAttr : LLVM_Attr<"DIStringType", "di_string_type",
+                                     /*traits=*/[], "DITypeAttr"> {
+  let parameters = (ins
+    LLVM_DITagParameter:$tag,
+    "StringAttr":$name,
+    OptionalParameter<"uint64_t">:$sizeInBits,
+    OptionalParameter<"uint32_t">:$alignInBits,
+    OptionalParameter<"DIVariableAttr">:$stringLength,
+    OptionalParameter<"DIExpressionAttr">:$stringLengthExp,
+    OptionalParameter<"DIExpressionAttr">:$stringLocationExp,
+    LLVM_DIEncodingParameter:$encoding
+  );
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
 //===----------------------------------------------------------------------===//
 // MemoryEffectsAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
index 091a817caf3d6e0f76922d6ac801e042720b6d8b..3a93be21da37561e9e31a10e61dbf77a2879033a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
@@ -60,6 +60,15 @@ public:
   static bool classof(Attribute attr);
 };
 
+/// This class represents a LLVM attribute that describes a debug info variable.
+class DIVariableAttr : public DINodeAttr {
+public:
+  using DINodeAttr::DINodeAttr;
+
+  /// Support LLVM type casting.
+  static bool classof(Attribute attr);
+};
+
 /// Base class for LLVM attributes participating in the TBAA graph.
 class TBAANodeAttr : public Attribute {
 public:
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index a93964abcb42e7b80b5573643d81405f84bf0d66..f8e85004d5f93c8a83b7134c06a970bd77abbc89 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -716,12 +716,15 @@ def FramePointerKindNonLeaf
     : LLVM_EnumAttrCase<"NonLeaf", "non-leaf", "NonLeaf", 1>;
 def FramePointerKindAll
     : LLVM_EnumAttrCase<"All", "all", "All", 2>;
+def FramePointerKindReserved
+    : LLVM_EnumAttrCase<"Reserved", "reserved", "Reserved", 3>;
 
 def FramePointerKindEnum : LLVM_EnumAttr<
     "FramePointerKind",
     "::llvm::FramePointerKind",
     "LLVM FramePointerKind",
-    [FramePointerKindNone, FramePointerKindNonLeaf, FramePointerKindAll]> {
+    [FramePointerKindNone, FramePointerKindNonLeaf,
+     FramePointerKindAll, FramePointerKindReserved]> {
   let cppNamespace = "::mlir::LLVM::framePointerKind";
 }
 
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index 0b063aa772bab038e650f840a33c6159ccd113fd..f35ea962bea168b1bb8feb2d283642c59805e7ad 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -136,8 +136,8 @@ def ExecuteRegionOp : SCF_Op<"execute_region", [
 def ForOp : SCF_Op<"for",
       [AutomaticAllocationScope, DeclareOpInterfaceMethods<LoopLikeOpInterface,
        ["getInitsMutable", "getLoopResults", "getRegionIterArgs",
-        "getSingleInductionVar", "getSingleLowerBound", "getSingleStep",
-        "getSingleUpperBound", "getYieldedValuesMutable",
+        "getLoopInductionVars", "getLoopLowerBounds", "getLoopSteps",
+        "getLoopUpperBounds", "getYieldedValuesMutable",
         "promoteIfSingleIteration", "replaceWithAdditionalYields",
         "yieldTiledValuesAndReplace"]>,
        AllTypesMatch<["lowerBound", "upperBound", "step"]>,
@@ -301,8 +301,8 @@ def ForallOp : SCF_Op<"forall", [
        AttrSizedOperandSegments,
        AutomaticAllocationScope,
        DeclareOpInterfaceMethods<LoopLikeOpInterface,
-          ["getInitsMutable", "getRegionIterArgs", "getSingleInductionVar", 
-           "getSingleLowerBound", "getSingleUpperBound", "getSingleStep",
+          ["getInitsMutable", "getRegionIterArgs", "getLoopInductionVars", 
+           "getLoopLowerBounds", "getLoopUpperBounds", "getLoopSteps",
            "promoteIfSingleIteration", "yieldTiledValuesAndReplace"]>,
        RecursiveMemoryEffects,
        SingleBlockImplicitTerminator<"scf::InParallelOp">,
@@ -510,22 +510,31 @@ def ForallOp : SCF_Op<"forall", [
   ];
 
   let extraClassDeclaration = [{
-    // Get lower bounds as OpFoldResult.
+    /// Get induction variables.
+    SmallVector<Value> getInductionVars() {
+      std::optional<SmallVector<Value>> maybeInductionVars = getLoopInductionVars();
+      assert(maybeInductionVars.has_value() && "expected values");
+      return *maybeInductionVars;
+    }
+    /// Get lower bounds as OpFoldResult.
     SmallVector<OpFoldResult> getMixedLowerBound() {
-      Builder b(getOperation()->getContext());
-      return getMixedValues(getStaticLowerBound(), getDynamicLowerBound(), b);
+      std::optional<SmallVector<OpFoldResult>> maybeLowerBounds = getLoopLowerBounds();
+      assert(maybeLowerBounds.has_value() && "expected values");
+      return *maybeLowerBounds;
     }
 
-    // Get upper bounds as OpFoldResult.
+    /// Get upper bounds as OpFoldResult.
     SmallVector<OpFoldResult> getMixedUpperBound() {
-      Builder b(getOperation()->getContext());
-      return getMixedValues(getStaticUpperBound(), getDynamicUpperBound(), b);
+      std::optional<SmallVector<OpFoldResult>> maybeUpperBounds = getLoopUpperBounds();
+      assert(maybeUpperBounds.has_value() && "expected values");
+      return *maybeUpperBounds;
     }
 
-    // Get steps as OpFoldResult.
+    /// Get steps as OpFoldResult.
     SmallVector<OpFoldResult> getMixedStep() {
-      Builder b(getOperation()->getContext());
-      return getMixedValues(getStaticStep(), getDynamicStep(), b);
+      std::optional<SmallVector<OpFoldResult>> maybeSteps = getLoopSteps();
+      assert(maybeSteps.has_value() && "expected values");
+      return *maybeSteps;
     }
 
     /// Get lower bounds as values.
@@ -584,10 +593,6 @@ def ForallOp : SCF_Op<"forall", [
                                     getNumDynamicControlOperands() + getRank());
     }
 
-    ::mlir::ValueRange getInductionVars() {
-      return getBody()->getArguments().take_front(getRank());
-    }
-
     ::mlir::Value getInductionVar(int64_t idx) {
       return getInductionVars()[idx];
     }
@@ -765,8 +770,8 @@ def IfOp : SCF_Op<"if", [DeclareOpInterfaceMethods<RegionBranchOpInterface, [
 def ParallelOp : SCF_Op<"parallel",
     [AutomaticAllocationScope,
      AttrSizedOperandSegments,
-     DeclareOpInterfaceMethods<LoopLikeOpInterface, ["getSingleInductionVar",
-          "getSingleLowerBound", "getSingleUpperBound", "getSingleStep"]>,
+     DeclareOpInterfaceMethods<LoopLikeOpInterface, ["getLoopInductionVars",
+          "getLoopLowerBounds", "getLoopUpperBounds", "getLoopSteps"]>,
      RecursiveMemoryEffects,
      DeclareOpInterfaceMethods<RegionBranchOpInterface>,
      SingleBlockImplicitTerminator<"scf::ReduceOp">,
@@ -846,8 +851,11 @@ def ParallelOp : SCF_Op<"parallel",
   ];
 
   let extraClassDeclaration = [{
-    ValueRange getInductionVars() {
-      return getBody()->getArguments();
+    /// Get induction variables.
+    SmallVector<Value> getInductionVars() {
+      std::optional<SmallVector<Value>> maybeInductionVars = getLoopInductionVars();;
+      assert(maybeInductionVars.has_value() && "expected values");
+      return *maybeInductionVars;
     }
     unsigned getNumLoops() { return getStep().size(); }
     unsigned getNumReductions() { return getInitVals().size(); }
diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
index fea5afa0b7bb39531c18764d3ef660d0819390f1..81bab1b0c82f7a100dde29415d933846d0a340ee 100644
--- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
@@ -114,12 +114,14 @@ def ApplyReassociativeReshapeFoldingPatternsOp : Op<Transform_Dialect,
 def ApplyRewriteTensorOpsAsConstantPatternsOp : Op<Transform_Dialect,
     "apply_patterns.tensor.rewrite_as_constant",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let arguments = (ins UnitAttr:$aggressive);
   let description = [{
     Indicates that tensor ops (such as tensor.generate) should be replaced with
     constants (arith.constant) when possible.
   }];
 
-  let assemblyFormat = "attr-dict";
+  let assemblyFormat =
+      "(`aggressive` $aggressive^)? attr-dict";
 }
 
 def Transform_TensorPadOp : Transform_ConcreteOpType<"tensor.pad">;
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
index 7dabc266c023b7f74a2e3459df2b7797e17f4bc3..7f983b8b3cfd06a8efbabeab888c32af9ec5124c 100644
--- a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
@@ -91,9 +91,12 @@ void populateSimplifyPackAndUnpackPatterns(RewritePatternSet &patterns);
 /// respectively.
 void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns);
 
+using ControlFoldFn = std::function<bool(OpOperand *)>;
+
 /// Populates `patterns` with patterns that replace tensor ops (such as
 /// tensor.generate) with constants when possible.
-void populateRewriteAsConstantPatterns(RewritePatternSet &patterns);
+void populateRewriteAsConstantPatterns(RewritePatternSet &patterns,
+                                       const ControlFoldFn &controlFn);
 
 //===----------------------------------------------------------------------===//
 // Transform helpers
diff --git a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
index e8f6edc3f133e1feaf3d115ed1720f3bec465a23..89bc57f09ec8ba3b4974420b0bba39534a707bbb 100644
--- a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
@@ -85,21 +85,49 @@ bool isReassociationValid(ArrayRef<AffineMap> reassociation,
 template <typename ReshapeOpTy, typename InverseReshapeOpTy>
 static OpFoldResult foldReshapeOp(ReshapeOpTy reshapeOp,
                                   ArrayRef<Attribute> operands) {
-
+  // Fold identity reshape.
   if (reshapeOp.getSrcType() == reshapeOp.getType())
     return reshapeOp.getSrc();
 
-  // Fold producer-consumer reshape ops where the operand type of the
-  // producer is same as the return type of the consumer.
-  auto reshapeSrcOp =
-      reshapeOp.getSrc().template getDefiningOp<InverseReshapeOpTy>();
-  if (reshapeSrcOp && reshapeSrcOp.getSrcType() == reshapeOp.getResultType())
-    return reshapeSrcOp.getSrc();
-
   // Reshape of a constant can be replaced with a new constant.
   if (auto elements = dyn_cast_or_null<DenseElementsAttr>(operands.front()))
     return elements.reshape(cast<ShapedType>(reshapeOp.getResult().getType()));
 
+  // Fold if the producer reshape source has the same shape with at most 1
+  // dynamic dimension.
+  auto reshapeSrcOp =
+      reshapeOp.getSrc().template getDefiningOp<InverseReshapeOpTy>();
+  if (!reshapeSrcOp)
+    return nullptr;
+  auto srcType = reshapeSrcOp.getSrcType();
+  auto resultType = reshapeOp.getResultType();
+  if (srcType != resultType)
+    return nullptr;
+
+  if (llvm::count_if(srcType.getShape(), ShapedType::isDynamic) < 2) {
+    return reshapeSrcOp.getSrc();
+  }
+
+  // Fold producer-consumer reshape ops when they are perfect inverses of each
+  // other:
+  //   1) Reassociation indices are equivalent.
+  //   2) Boundary types are equivalent.
+  //   3) No reassociations have more than 1 dynamic dimension, and reassociated
+  //      shapes are equal for each reassociation.
+  auto reassociations = reshapeOp.getReassociationIndices();
+  if (reassociations != reshapeSrcOp.getReassociationIndices())
+    return nullptr;
+  // If the reshapes are expanding and then collapsing, the ops can be folded
+  // despite multiple dynamic dimensions.
+  if (srcType.getRank() < reshapeSrcOp.getResultType().getRank())
+    return reshapeSrcOp.getSrc();
+  if (llvm::all_of(reassociations, [&](auto reInd) {
+        ArrayRef<int64_t> srcSlice =
+            srcType.getShape().slice(reInd.front(), reInd.size());
+        return llvm::count_if(srcSlice, ShapedType::isDynamic) < 2;
+      })) {
+    return reshapeSrcOp.getSrc();
+  }
   return nullptr;
 }
 
@@ -360,10 +388,12 @@ private:
           resultShape.slice(resultIndices.front(), resultIndices.size());
 
       if (srcSubShape.size() == resultSubShape.size()) {
-        if (srcSubShape == resultSubShape)
+        if (srcSubShape == resultSubShape &&
+            llvm::count_if(srcSubShape, ShapedType::isDynamic) < 2) {
           composedReassociation.push_back(srcIndices);
-        else
+        } else {
           return std::nullopt;
+        }
       }
 
       // Find reassociation to collapse `srcSubShape` into `resultSubShape`.
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index 2562301e499dd955624894e1623de35c5ce0c375..ed7b9ece4a464bfb5e04b93a91df987f9c6d247e 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -784,6 +784,7 @@ public:
 /// place.
 class PatternRewriter : public RewriterBase {
 public:
+  explicit PatternRewriter(MLIRContext *ctx) : RewriterBase(ctx) {}
   using RewriterBase::RewriterBase;
 
   /// A hook used to indicate if the pattern rewriter can recover from failure
diff --git a/mlir/include/mlir/Interfaces/LoopLikeInterface.td b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
index f0dc6e60eba58424abb78bc812a40a2755a45929..b748d5e29114a18ce3a40448e17fabaf4681c804 100644
--- a/mlir/include/mlir/Interfaces/LoopLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
@@ -93,51 +93,59 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
       }]
     >,
     InterfaceMethod<[{
-        If there is a single induction variable return it, otherwise return
-        std::nullopt.
+        Return all induction variables, if they exist. If the op has no notion of
+        induction variable, then return std::nullopt. If it does have
+        a notion but an instance doesn't have induction variables, then
+        return empty vector.
       }],
-      /*retTy=*/"::std::optional<::mlir::Value>",
-      /*methodName=*/"getSingleInductionVar",
+      /*retTy=*/"::std::optional<::llvm::SmallVector<::mlir::Value>>",
+      /*methodName=*/"getLoopInductionVars",
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return std::nullopt;
+        return ::std::nullopt;
       }]
     >,
     InterfaceMethod<[{
-        Return the single lower bound value or attribute if it exists, otherwise
-        return std::nullopt.
+        Return all lower bounds, if they exist. If the op has no notion of
+        lower bounds, then return std::nullopt. If it does have
+        a notion but an instance doesn't have lower bounds, then
+        return empty vector.
       }],
-      /*retTy=*/"::std::optional<::mlir::OpFoldResult>",
-      /*methodName=*/"getSingleLowerBound",
+      /*retTy=*/"::std::optional<::llvm::SmallVector<::mlir::OpFoldResult>>",
+      /*methodName=*/"getLoopLowerBounds",
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return std::nullopt;
+        return ::std::nullopt;
       }]
     >,
     InterfaceMethod<[{
-        Return the single step value or attribute if it exists, otherwise
-        return std::nullopt.
+        Return all steps, if they exist. If the op has no notion of
+        steps, then return std::nullopt. If it does have
+        a notion but an instance doesn't have steps, then
+        return empty vector.
       }],
-      /*retTy=*/"::std::optional<::mlir::OpFoldResult>",
-      /*methodName=*/"getSingleStep",
+      /*retTy=*/"::std::optional<::llvm::SmallVector<::mlir::OpFoldResult>>",
+      /*methodName=*/"getLoopSteps",
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return std::nullopt;
+        return ::std::nullopt;
       }]
     >,
     InterfaceMethod<[{
-        Return the single upper bound value or attribute if it exists, otherwise
-        return std::nullopt.
+        Return all upper bounds, if they exist. If the op has no notion of
+        lower bounds, then return std::nullopt. If it does have
+        a notion but an instance doesn't have lower bounds, then
+        return empty vector.
       }],
-      /*retTy=*/"::std::optional<::mlir::OpFoldResult>",
-      /*methodName=*/"getSingleUpperBound",
+      /*retTy=*/"::std::optional<::llvm::SmallVector<::mlir::OpFoldResult>>",
+      /*methodName=*/"getLoopUpperBounds",
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return std::nullopt;
+        return ::std::nullopt;
       }]
     >,
     InterfaceMethod<[{
@@ -235,6 +243,39 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
   }];
 
   let extraSharedClassDeclaration = [{
+    /// If there is a single induction variable return it, otherwise return
+    /// std::nullopt.
+    ::std::optional<::mlir::Value> getSingleInductionVar() {
+      auto inductionVars = this->getLoopInductionVars();
+      if (inductionVars.has_value() && (*inductionVars).size() == 1)
+          return (*inductionVars)[0];
+        return std::nullopt;
+    }
+    /// Return the single lower bound value or attribute if it exists, otherwise
+    /// return std::nullopt.
+    ::std::optional<::mlir::OpFoldResult> getSingleLowerBound() {
+      auto lowerBounds = this->getLoopLowerBounds();
+      if (lowerBounds.has_value() && (*lowerBounds).size() == 1)
+          return (*lowerBounds)[0];
+      return std::nullopt;
+    }
+    /// Return the single step value or attribute if it exists, otherwise
+    /// return std::nullopt.
+    ::std::optional<::mlir::OpFoldResult> getSingleStep() {
+      auto steps = this->getLoopSteps(); 
+      if (steps.has_value() && (*steps).size() == 1)
+          return (*steps)[0];
+      return std::nullopt;
+    }
+    /// Return the single upper bound value or attribute if it exists, otherwise
+    /// return std::nullopt.
+    ::std::optional<::mlir::OpFoldResult> getSingleUpperBound() {
+      auto upperBounds = this->getLoopUpperBounds();
+      if (upperBounds.has_value() && (*upperBounds).size() == 1)
+          return (*upperBounds)[0];
+      return std::nullopt;
+    }
+
     /// Append the specified additional "init" operands: replace this loop with
     /// a new loop that has the additional init operands. The loop body of this
     /// loop is moved over to the new loop.
diff --git a/mlir/include/mlir/Target/LLVMIR/Export.h b/mlir/include/mlir/Target/LLVMIR/Export.h
index 224496865513858d39a40e47913c0611cdd83624..893aaaa4faff6a9d4bf43ec948567433cd1fd8b7 100644
--- a/mlir/include/mlir/Target/LLVMIR/Export.h
+++ b/mlir/include/mlir/Target/LLVMIR/Export.h
@@ -20,10 +20,11 @@ class Module;
 namespace mlir {
 class Operation;
 
-/// Translate operation that satisfies LLVM dialect module requirements into an
-/// LLVM IR module living in the given context. This translates operations from
-/// any dilalect that has a registered implementation of
-/// LLVMTranslationDialectInterface.
+/// Translates a given LLVM dialect `module` into an LLVM IR module living in
+/// the given context. Operates on any operation from dialects that provide a
+/// registered implementation of the LLVMTranslationDialectInterface. Returns
+/// nullptr when the translation fails.
+/// Verifies the produced LLVM module, except when `disableVerification` is set.
 std::unique_ptr<llvm::Module>
 translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
                         llvm::StringRef name = "LLVMDialectModule",
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 83198c9b0db545561d98c3758efa5868c1dd3b80..f6c51499f271c54050d46bbfc8b230c46d26a8bc 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -1088,8 +1088,9 @@ struct ConversionConfig {
 
   /// An optional listener that is notified about all IR modifications in case
   /// dialect conversion succeeds. If the dialect conversion fails and no IR
-  /// modifications are visible (i.e., they were all rolled back), no
-  /// notifications are sent.
+  /// modifications are visible (i.e., they were all rolled back), or if the
+  /// dialect conversion is an "analysis conversion", no notifications are
+  /// sent (apart from `notifyPatternBegin`/notifyPatternEnd`).
   ///
   /// Note: Notifications are sent in a delayed fashion, when the dialect
   /// conversion is guaranteed to succeed. At that point, some IR modifications
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index de20632b4fb7d600b6b971c40f1c4fb76d232afb..4b6b54dc1c1115ae491b6d3041e612c4efd0361c 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -3238,6 +3238,19 @@ void mlir::python::populateIRCore(py::module &m) {
             return PyBlockArgumentList(self.getParentOperation(), self.get());
           },
           "Returns a list of block arguments.")
+      .def(
+          "add_argument",
+          [](PyBlock &self, const PyType &type, const PyLocation &loc) {
+            return mlirBlockAddArgument(self.get(), type, loc);
+          },
+          "Append an argument of the specified type to the block and returns "
+          "the newly added argument.")
+      .def(
+          "erase_argument",
+          [](PyBlock &self, unsigned index) {
+            return mlirBlockEraseArgument(self.get(), index);
+          },
+          "Erase the argument at 'index' and remove it from the argument list.")
       .def_property_readonly(
           "operations",
           [](PyBlock &self) {
diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp
index f6fb2cbedfac0ccccda36633ea6bc9bf08ca6502..754c94511524dbada359be12645a915bf7cc0a2a 100644
--- a/mlir/lib/CAPI/Dialect/LLVM.cpp
+++ b/mlir/lib/CAPI/Dialect/LLVM.cpp
@@ -195,6 +195,18 @@ MlirAttribute mlirLLVMDIDerivedTypeAttrGet(
       addressSpace, cast<DINodeAttr>(unwrap(extraData))));
 }
 
+MlirAttribute mlirLLVMDIStringTypeAttrGet(
+    MlirContext ctx, unsigned int tag, MlirAttribute name, uint64_t sizeInBits,
+    uint32_t alignInBits, MlirAttribute stringLength,
+    MlirAttribute stringLengthExp, MlirAttribute stringLocationExp,
+    MlirLLVMTypeEncoding encoding) {
+  return wrap(DIStringTypeAttr::get(
+      unwrap(ctx), tag, cast<StringAttr>(unwrap(name)), sizeInBits, alignInBits,
+      cast<DIVariableAttr>(unwrap(stringLength)),
+      cast<DIExpressionAttr>(unwrap(stringLengthExp)),
+      cast<DIExpressionAttr>(unwrap(stringLocationExp)), encoding));
+}
+
 MlirAttribute
 mlirLLVMDIDerivedTypeAttrGetBaseType(MlirAttribute diDerivedType) {
   return wrap(cast<DIDerivedTypeAttr>(unwrap(diDerivedType)).getBaseType());
diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp
index a72cd247e73f6077703ca89f34da2991d5ff5d8b..4e823c866f4337801e9bf422a2c207ce4dbb3344 100644
--- a/mlir/lib/CAPI/IR/IR.cpp
+++ b/mlir/lib/CAPI/IR/IR.cpp
@@ -906,6 +906,10 @@ MlirValue mlirBlockAddArgument(MlirBlock block, MlirType type,
   return wrap(unwrap(block)->addArgument(unwrap(type), unwrap(loc)));
 }
 
+void mlirBlockEraseArgument(MlirBlock block, unsigned index) {
+  return unwrap(block)->eraseArgument(index);
+}
+
 MlirValue mlirBlockInsertArgument(MlirBlock block, intptr_t pos, MlirType type,
                                   MlirLocation loc) {
   return wrap(unwrap(block)->insertArgument(pos, unwrap(type), unwrap(loc)));
diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
index 6a8ecb7b00473a4fd59be67c852ee0c2d2dfca79..53b79839da04c39082d75fb69005d83ee6fc7482 100644
--- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
+++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitC.cpp
@@ -31,15 +31,14 @@ public:
   LogicalResult
   matchAndRewrite(func::CallOp callOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    // Multiple results func was not converted to `emitc.func`.
+    // Multiple results func cannot be converted to `emitc.func`.
     if (callOp.getNumResults() > 1)
       return rewriter.notifyMatchFailure(
           callOp, "only functions with zero or one result can be converted");
 
-    rewriter.replaceOpWithNewOp<emitc::CallOp>(
-        callOp,
-        callOp.getNumResults() ? callOp.getResult(0).getType() : nullptr,
-        adaptor.getOperands(), callOp->getAttrs());
+    rewriter.replaceOpWithNewOp<emitc::CallOp>(callOp, callOp.getResultTypes(),
+                                               adaptor.getOperands(),
+                                               callOp->getAttrs());
 
     return success();
   }
diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
index b29abc94ce400c4af5b6834df9f808f7c1b09230..e48ca5180b706ffa60a6e53d53aa5820f4ceb294 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
@@ -10,18 +10,14 @@
 #include "mlir/Analysis/DataLayoutAnalysis.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/SymbolTable.h"
 
 using namespace mlir;
 
 namespace {
-// TODO: Fix the LLVM utilities for looking up functions to take Operation*
-// with SymbolTable trait instead of ModuleOp and make similar change here. This
-// allows call sites to use getParentWithTrait<OpTrait::SymbolTable> instead
-// of getParentOfType<ModuleOp> to pass down the operation.
 LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter,
-                                      ModuleOp module, Type indexType) {
+                                      Operation *module, Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
-
   if (useGenericFn)
     return LLVM::lookupOrCreateGenericAllocFn(module, indexType);
 
@@ -29,7 +25,7 @@ LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter,
 }
 
 LLVM::LLVMFuncOp getAlignedAllocFn(const LLVMTypeConverter *typeConverter,
-                                   ModuleOp module, Type indexType) {
+                                   Operation *module, Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
 
   if (useGenericFn)
@@ -79,7 +75,8 @@ std::tuple<Value, Value> AllocationOpLLVMLowering::allocateBufferManuallyAlign(
   // Allocate the underlying buffer.
   Type elementPtrType = this->getElementPtrType(memRefType);
   LLVM::LLVMFuncOp allocFuncOp = getNotalignedAllocFn(
-      getTypeConverter(), op->getParentOfType<ModuleOp>(), getIndexType());
+      getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
+      getIndexType());
   auto results = rewriter.create<LLVM::CallOp>(loc, allocFuncOp, sizeBytes);
 
   Value allocatedPtr =
@@ -144,7 +141,8 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign(
 
   Type elementPtrType = this->getElementPtrType(memRefType);
   LLVM::LLVMFuncOp allocFuncOp = getAlignedAllocFn(
-      getTypeConverter(), op->getParentOfType<ModuleOp>(), getIndexType());
+      getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
+      getIndexType());
   auto results = rewriter.create<LLVM::CallOp>(
       loc, allocFuncOp, ValueRange({allocAlignment, sizeBytes}));
 
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 367142a520742778da633c72857ec5041ec518dd..0a8924222525546423856e7c84ca32c7f0b71203 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -102,10 +102,8 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   // assigned to by emitc::assign ops within the loop body.
   SmallVector<Value> resultVariables =
       createVariablesForResults(forOp, rewriter);
-  SmallVector<Value> iterArgsVariables =
-      createVariablesForResults(forOp, rewriter);
 
-  assignValues(forOp.getInits(), iterArgsVariables, rewriter, loc);
+  assignValues(forOp.getInits(), resultVariables, rewriter, loc);
 
   emitc::ForOp loweredFor = rewriter.create<emitc::ForOp>(
       loc, forOp.getLowerBound(), forOp.getUpperBound(), forOp.getStep());
@@ -117,15 +115,12 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
 
   SmallVector<Value> replacingValues;
   replacingValues.push_back(loweredFor.getInductionVar());
-  replacingValues.append(iterArgsVariables.begin(), iterArgsVariables.end());
+  replacingValues.append(resultVariables.begin(), resultVariables.end());
 
   rewriter.mergeBlocks(forOp.getBody(), loweredBody, replacingValues);
-  lowerYield(iterArgsVariables, rewriter,
+  lowerYield(resultVariables, rewriter,
              cast<scf::YieldOp>(loweredBody->getTerminator()));
 
-  // Copy iterArgs into results after the for loop.
-  assignValues(iterArgsVariables, resultVariables, rewriter, loc);
-
   rewriter.replaceOp(forOp, resultVariables);
   return success();
 }
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 2e31487bd55a0a945e1809f5aa70f90ddb1a5b53..0a58d2fdb02f5c18ce0e38a87f3b8181d71c8d82 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -2454,27 +2454,30 @@ bool AffineForOp::matchingBoundOperandList() {
 
 SmallVector<Region *> AffineForOp::getLoopRegions() { return {&getRegion()}; }
 
-std::optional<Value> AffineForOp::getSingleInductionVar() {
-  return getInductionVar();
+std::optional<SmallVector<Value>> AffineForOp::getLoopInductionVars() {
+  return SmallVector<Value>{getInductionVar()};
 }
 
-std::optional<OpFoldResult> AffineForOp::getSingleLowerBound() {
+std::optional<SmallVector<OpFoldResult>> AffineForOp::getLoopLowerBounds() {
   if (!hasConstantLowerBound())
     return std::nullopt;
   OpBuilder b(getContext());
-  return OpFoldResult(b.getI64IntegerAttr(getConstantLowerBound()));
+  return SmallVector<OpFoldResult>{
+      OpFoldResult(b.getI64IntegerAttr(getConstantLowerBound()))};
 }
 
-std::optional<OpFoldResult> AffineForOp::getSingleStep() {
+std::optional<SmallVector<OpFoldResult>> AffineForOp::getLoopSteps() {
   OpBuilder b(getContext());
-  return OpFoldResult(b.getI64IntegerAttr(getStepAsInt()));
+  return SmallVector<OpFoldResult>{
+      OpFoldResult(b.getI64IntegerAttr(getStepAsInt()))};
 }
 
-std::optional<OpFoldResult> AffineForOp::getSingleUpperBound() {
+std::optional<SmallVector<OpFoldResult>> AffineForOp::getLoopUpperBounds() {
   if (!hasConstantUpperBound())
-    return std::nullopt;
+    return {};
   OpBuilder b(getContext());
-  return OpFoldResult(b.getI64IntegerAttr(getConstantUpperBound()));
+  return SmallVector<OpFoldResult>{
+      OpFoldResult(b.getI64IntegerAttr(getConstantUpperBound()))};
 }
 
 FailureOr<LoopLikeOpInterface> AffineForOp::replaceWithAdditionalYields(
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/EnableArmStreaming.cpp b/mlir/lib/Dialect/ArmSME/Transforms/EnableArmStreaming.cpp
index 79a6caffb6ee0bfd0c843d5d378016f20ac914a7..fb4bb41d87488e0ba6eddf4faea6ea5ea13715d2 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/EnableArmStreaming.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/EnableArmStreaming.cpp
@@ -58,17 +58,25 @@ constexpr StringLiteral
 struct EnableArmStreamingPass
     : public arm_sme::impl::EnableArmStreamingBase<EnableArmStreamingPass> {
   EnableArmStreamingPass(ArmStreamingMode streamingMode, ArmZaMode zaMode,
-                         bool onlyIfRequiredByOps) {
+                         bool ifRequiredByOps, bool ifContainsScalableVectors) {
     this->streamingMode = streamingMode;
     this->zaMode = zaMode;
-    this->onlyIfRequiredByOps = onlyIfRequiredByOps;
+    this->ifRequiredByOps = ifRequiredByOps;
+    this->ifContainsScalableVectors = ifContainsScalableVectors;
   }
   void runOnOperation() override {
-    auto op = getOperation();
+    auto function = getOperation();
 
-    if (onlyIfRequiredByOps) {
+    if (ifRequiredByOps && ifContainsScalableVectors) {
+      function->emitOpError(
+          "enable-arm-streaming: `if-required-by-ops` and "
+          "`if-contains-scalable-vectors` are mutually exclusive");
+      return signalPassFailure();
+    }
+
+    if (ifRequiredByOps) {
       bool foundTileOp = false;
-      op.walk([&](Operation *op) {
+      function.walk([&](Operation *op) {
         if (llvm::isa<ArmSMETileOpInterface>(op)) {
           foundTileOp = true;
           return WalkResult::interrupt();
@@ -79,27 +87,46 @@ struct EnableArmStreamingPass
         return;
     }
 
-    if (op->getAttr(kEnableArmStreamingIgnoreAttr) ||
+    if (ifContainsScalableVectors) {
+      bool foundScalableVector = false;
+      auto isScalableVector = [&](Type type) {
+        if (auto vectorType = dyn_cast<VectorType>(type))
+          return vectorType.isScalable();
+        return false;
+      };
+      function.walk([&](Operation *op) {
+        if (llvm::any_of(op->getOperandTypes(), isScalableVector) ||
+            llvm::any_of(op->getResultTypes(), isScalableVector)) {
+          foundScalableVector = true;
+          return WalkResult::interrupt();
+        }
+        return WalkResult::advance();
+      });
+      if (!foundScalableVector)
+        return;
+    }
+
+    if (function->getAttr(kEnableArmStreamingIgnoreAttr) ||
         streamingMode == ArmStreamingMode::Disabled)
       return;
 
     auto unitAttr = UnitAttr::get(&getContext());
 
-    op->setAttr(stringifyArmStreamingMode(streamingMode), unitAttr);
+    function->setAttr(stringifyArmStreamingMode(streamingMode), unitAttr);
 
     // The pass currently only supports enabling ZA when in streaming-mode, but
     // ZA can be accessed by the SME LDR, STR and ZERO instructions when not in
     // streaming-mode (see section B1.1.1, IDGNQM of spec [1]). It may be worth
     // supporting this later.
     if (zaMode != ArmZaMode::Disabled)
-      op->setAttr(stringifyArmZaMode(zaMode), unitAttr);
+      function->setAttr(stringifyArmZaMode(zaMode), unitAttr);
   }
 };
 } // namespace
 
 std::unique_ptr<Pass> mlir::arm_sme::createEnableArmStreamingPass(
     const ArmStreamingMode streamingMode, const ArmZaMode zaMode,
-    bool onlyIfRequiredByOps) {
-  return std::make_unique<EnableArmStreamingPass>(streamingMode, zaMode,
-                                                  onlyIfRequiredByOps);
+    bool ifRequiredByOps, bool ifContainsScalableVectors) {
+  return std::make_unique<EnableArmStreamingPass>(
+      streamingMode, zaMode, ifRequiredByOps, ifContainsScalableVectors);
 }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index a8ec111f8c304ba22ece94a952bad8590bc4d22e..bd5c4d476921696c4170998596410fba69ff4ff3 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -822,10 +822,11 @@ FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
 
   // Lookup the function operation and check if it has private visibility. If
   // the function is referenced by SSA value instead of a Symbol, it's assumed
-  // to be always private.
+  // to be public. (And we cannot easily change the type of the SSA value
+  // anyway.)
   Operation *funcOp = op.resolveCallable(state.getSymbolTable());
-  bool isPrivate = true;
-  if (auto symbol = dyn_cast<SymbolOpInterface>(funcOp))
+  bool isPrivate = false;
+  if (auto symbol = dyn_cast_or_null<SymbolOpInterface>(funcOp))
     isPrivate = symbol.isPrivate() && !symbol.isDeclaration();
 
   // If the private-function-dynamic-ownership option is enabled and we are
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 0c2590d711301b56e394d912375df55f678fe3ab..d8e29da6512d4ed5899a8d10b02b7ce36a164c5c 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -1730,12 +1730,11 @@ void GPUModuleOp::build(OpBuilder &builder, OperationState &result,
                         StringRef name, ArrayAttr targets,
                         Attribute offloadingHandler) {
   ensureTerminator(*result.addRegion(), builder, result.location);
-  result.attributes.push_back(builder.getNamedAttr(
-      ::mlir::SymbolTable::getSymbolAttrName(), builder.getStringAttr(name)));
 
   Properties &props = result.getOrAddProperties<Properties>();
   if (targets)
     props.targets = targets;
+  props.setSymName(builder.getStringAttr(name));
   props.offloadingHandler = offloadingHandler;
 }
 
@@ -1751,11 +1750,11 @@ ParseResult GPUModuleOp::parse(OpAsmParser &parser, OperationState &result) {
   StringAttr nameAttr;
   ArrayAttr targetsAttr;
 
-  if (parser.parseSymbolName(nameAttr, mlir::SymbolTable::getSymbolAttrName(),
-                             result.attributes))
+  if (parser.parseSymbolName(nameAttr))
     return failure();
 
   Properties &props = result.getOrAddProperties<Properties>();
+  props.setSymName(nameAttr);
 
   // Parse the optional offloadingHandler
   if (succeeded(parser.parseOptionalLess())) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 0004c2e3403e53a78731fa5c5766a6f8c11b2991..88421a16ccf9fba1252edf1bd30808431e12c822 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -45,49 +45,53 @@ static constexpr llvm::StringRef kGenericFree = "_mlir_memref_to_llvm_free";
 static constexpr llvm::StringRef kMemRefCopy = "memrefCopy";
 
 /// Generic print function lookupOrCreate helper.
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFn(ModuleOp moduleOp, StringRef name,
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFn(Operation *moduleOp,
+                                              StringRef name,
                                               ArrayRef<Type> paramTypes,
                                               Type resultType, bool isVarArg) {
-  auto func = moduleOp.lookupSymbol<LLVM::LLVMFuncOp>(name);
+  assert(moduleOp->hasTrait<OpTrait::SymbolTable>() &&
+         "expected SymbolTable operation");
+  auto func = llvm::dyn_cast_or_null<LLVM::LLVMFuncOp>(
+      SymbolTable::lookupSymbolIn(moduleOp, name));
   if (func)
     return func;
-  OpBuilder b(moduleOp.getBodyRegion());
+  OpBuilder b(moduleOp->getRegion(0));
   return b.create<LLVM::LLVMFuncOp>(
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintI64Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintI64Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintI64,
                           IntegerType::get(moduleOp->getContext(), 64),
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintU64Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintU64Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintU64,
                           IntegerType::get(moduleOp->getContext(), 64),
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF16Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF16Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintF16,
                           IntegerType::get(moduleOp->getContext(), 16), // bits!
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintBF16Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintBF16Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintBF16,
                           IntegerType::get(moduleOp->getContext(), 16), // bits!
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF32Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF32Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintF32,
                           Float32Type::get(moduleOp->getContext()),
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintF64,
                           Float64Type::get(moduleOp->getContext()),
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
@@ -103,72 +107,72 @@ static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context) {
 }
 
 LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStringFn(
-    ModuleOp moduleOp, std::optional<StringRef> runtimeFunctionName) {
+    Operation *moduleOp, std::optional<StringRef> runtimeFunctionName) {
   return lookupOrCreateFn(moduleOp, runtimeFunctionName.value_or(kPrintString),
                           getCharPtr(moduleOp->getContext()),
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintOpen, {},
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCloseFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCloseFn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintClose, {},
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCommaFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCommaFn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintComma, {},
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(Operation *moduleOp) {
   return lookupOrCreateFn(moduleOp, kPrintNewline, {},
                           LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(Operation *moduleOp,
                                                     Type indexType) {
   return LLVM::lookupOrCreateFn(moduleOp, kMalloc, indexType,
                                 getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(Operation *moduleOp,
                                                           Type indexType) {
   return LLVM::lookupOrCreateFn(moduleOp, kAlignedAlloc, {indexType, indexType},
                                 getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(Operation *moduleOp) {
   return LLVM::lookupOrCreateFn(
       moduleOp, kFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(ModuleOp moduleOp,
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(Operation *moduleOp,
                                                           Type indexType) {
   return LLVM::lookupOrCreateFn(moduleOp, kGenericAlloc, indexType,
                                 getVoidPtr(moduleOp->getContext()));
 }
 
 LLVM::LLVMFuncOp
-mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp,
+mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp,
                                                 Type indexType) {
   return LLVM::lookupOrCreateFn(moduleOp, kGenericAlignedAlloc,
                                 {indexType, indexType},
                                 getVoidPtr(moduleOp->getContext()));
 }
 
-LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp) {
+LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(Operation *moduleOp) {
   return LLVM::lookupOrCreateFn(
       moduleOp, kGenericFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 LLVM::LLVMFuncOp
-mlir::LLVM::lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType,
+mlir::LLVM::lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
                                        Type unrankedDescriptorType) {
   return LLVM::lookupOrCreateFn(
       moduleOp, kMemRefCopy,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
index 9bc71e1ebc489dd5a404f137dfdc6c4eb8773e34..963a4be25079e3da1f88523723683ddeb5d606e4 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
@@ -60,8 +60,8 @@ bool DINodeAttr::classof(Attribute attr) {
                    DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr,
                    DILabelAttr, DILexicalBlockAttr, DILexicalBlockFileAttr,
                    DILocalVariableAttr, DIModuleAttr, DINamespaceAttr,
-                   DINullTypeAttr, DISubprogramAttr, DISubrangeAttr,
-                   DISubroutineTypeAttr>(attr);
+                   DINullTypeAttr, DIStringTypeAttr, DISubprogramAttr,
+                   DISubrangeAttr, DISubroutineTypeAttr>(attr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -82,13 +82,22 @@ bool DILocalScopeAttr::classof(Attribute attr) {
                    DISubprogramAttr>(attr);
 }
 
+//===----------------------------------------------------------------------===//
+// DIVariableAttr
+//===----------------------------------------------------------------------===//
+
+bool DIVariableAttr::classof(Attribute attr) {
+  return llvm::isa<DILocalVariableAttr, DIGlobalVariableAttr>(attr);
+}
+
 //===----------------------------------------------------------------------===//
 // DITypeAttr
 //===----------------------------------------------------------------------===//
 
 bool DITypeAttr::classof(Attribute attr) {
   return llvm::isa<DINullTypeAttr, DIBasicTypeAttr, DICompositeTypeAttr,
-                   DIDerivedTypeAttr, DISubroutineTypeAttr>(attr);
+                   DIDerivedTypeAttr, DIStringTypeAttr, DISubroutineTypeAttr>(
+      attr);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 60b911948d4a02018cb1ae8431a2ac85ed2ec645..76cac0b05b4754d6d998c19f8c624193e653a0d0 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3031,12 +3031,12 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface {
               DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr,
               DIGlobalVariableExpressionAttr, DILabelAttr, DILexicalBlockAttr,
               DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr,
-              DINamespaceAttr, DINullTypeAttr, DISubprogramAttr,
-              DISubroutineTypeAttr, LoopAnnotationAttr, LoopVectorizeAttr,
-              LoopInterleaveAttr, LoopUnrollAttr, LoopUnrollAndJamAttr,
-              LoopLICMAttr, LoopDistributeAttr, LoopPipelineAttr,
-              LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr, TBAATagAttr,
-              TBAATypeDescriptorAttr>([&](auto attr) {
+              DINamespaceAttr, DINullTypeAttr, DIStringTypeAttr,
+              DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr,
+              LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr,
+              LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr,
+              LoopPipelineAttr, LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr,
+              TBAATagAttr, TBAATypeDescriptorAttr>([&](auto attr) {
           os << decltype(attr)::getMnemonic();
           return AliasResult::OverridableAlias;
         })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index ad313c2d5ce603166acf2253ea46c93b607894f6..e73df61c9643416a0928e6c40230101de8881b11 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -956,6 +956,69 @@ private:
   ControlFusionFn controlFoldingReshapes;
 };
 
+class FoldPadWithProducerReshapeOpByExpansion
+    : public OpRewritePattern<tensor::PadOp> {
+public:
+  FoldPadWithProducerReshapeOpByExpansion(MLIRContext *context,
+                                          ControlFusionFn foldReshapes,
+                                          PatternBenefit benefit = 1)
+      : OpRewritePattern<tensor::PadOp>(context, benefit),
+        controlFoldingReshapes(std::move(foldReshapes)) {}
+
+  LogicalResult matchAndRewrite(tensor::PadOp padOp,
+                                PatternRewriter &rewriter) const override {
+    tensor::CollapseShapeOp reshapeOp =
+        padOp.getSource().getDefiningOp<tensor::CollapseShapeOp>();
+    if (!reshapeOp)
+      return failure();
+    if (!reshapeOp->hasOneUse())
+      return failure();
+
+    if (!controlFoldingReshapes(&padOp.getSourceMutable())) {
+      return rewriter.notifyMatchFailure(padOp,
+                                         "fusion blocked by control function");
+    }
+
+    ArrayRef<int64_t> low = padOp.getStaticLow();
+    ArrayRef<int64_t> high = padOp.getStaticHigh();
+    SmallVector<ReassociationIndices> reassociations =
+        reshapeOp.getReassociationIndices();
+
+    for (auto [reInd, l, h] : llvm::zip_equal(reassociations, low, high)) {
+      if (reInd.size() != 1 && (l != 0 || h != 0))
+        return failure();
+    }
+
+    SmallVector<OpFoldResult> newLow, newHigh;
+    RankedTensorType expandedType = reshapeOp.getSrcType();
+    RankedTensorType paddedType = padOp.getResultType();
+    SmallVector<int64_t> expandedPaddedShape(expandedType.getShape());
+    for (auto [idx, reInd] : llvm::enumerate(reassociations)) {
+      if (reInd.size() == 1) {
+        expandedPaddedShape[reInd[0]] = paddedType.getShape()[idx];
+      }
+      for (size_t i = 0; i < reInd.size(); ++i) {
+        newLow.push_back(padOp.getMixedLowPad()[idx]);
+        newHigh.push_back(padOp.getMixedHighPad()[idx]);
+      }
+    }
+
+    Location loc = padOp->getLoc();
+    RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape);
+    auto newPadOp = rewriter.create<tensor::PadOp>(
+        loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+        padOp.getConstantPaddingValue(), padOp.getNofold());
+
+    rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
+        padOp, padOp.getResultType(), newPadOp.getResult(), reassociations);
+
+    return success();
+  }
+
+private:
+  ControlFusionFn controlFoldingReshapes;
+};
+
 /// Pattern to fold a tensor.expand_shape op with its producer generic op
 /// by expanding the dimensionality of the loop in the producer op.
 struct FoldReshapeWithGenericOpByExpansion
@@ -1702,6 +1765,85 @@ private:
   ControlFusionFn controlFoldingReshapes;
 };
 
+class FoldPadWithProducerReshapeOpByCollapsing
+    : public OpRewritePattern<tensor::PadOp> {
+public:
+  FoldPadWithProducerReshapeOpByCollapsing(MLIRContext *context,
+                                           ControlFusionFn foldReshapes,
+                                           PatternBenefit benefit = 1)
+      : OpRewritePattern<tensor::PadOp>(context, benefit),
+        controlFoldingReshapes(std::move(foldReshapes)) {}
+
+  LogicalResult matchAndRewrite(tensor::PadOp padOp,
+                                PatternRewriter &rewriter) const override {
+    tensor::ExpandShapeOp reshapeOp =
+        padOp.getSource().getDefiningOp<tensor::ExpandShapeOp>();
+    if (!reshapeOp)
+      return failure();
+    if (!reshapeOp->hasOneUse())
+      return failure();
+
+    if (!controlFoldingReshapes(&padOp.getSourceMutable())) {
+      return rewriter.notifyMatchFailure(padOp,
+                                         "fusion blocked by control function");
+    }
+
+    ArrayRef<int64_t> low = padOp.getStaticLow();
+    ArrayRef<int64_t> high = padOp.getStaticHigh();
+    SmallVector<ReassociationIndices> reassociations =
+        reshapeOp.getReassociationIndices();
+
+    for (auto reInd : reassociations) {
+      if (reInd.size() == 1)
+        continue;
+      if (llvm::any_of(reInd, [&](int64_t ind) {
+            return low[ind] != 0 || high[ind] != 0;
+          })) {
+        return failure();
+      }
+    }
+
+    SmallVector<OpFoldResult> newLow, newHigh;
+    RankedTensorType collapsedType = reshapeOp.getSrcType();
+    RankedTensorType paddedType = padOp.getResultType();
+    SmallVector<int64_t> collapsedPaddedShape(collapsedType.getShape());
+    SmallVector<OpFoldResult> expandedPaddedSizes(
+        getMixedValues(reshapeOp.getStaticOutputShape(),
+                       reshapeOp.getOutputShape(), rewriter));
+    AffineExpr d0, d1, d2;
+    bindDims(rewriter.getContext(), d0, d1, d2);
+    auto addMap = AffineMap::get(3, 0, {d0 + d1 + d2});
+    Location loc = reshapeOp->getLoc();
+    for (auto [idx, reInd] : llvm::enumerate(reassociations)) {
+      OpFoldResult l = padOp.getMixedLowPad()[reInd[0]];
+      OpFoldResult h = padOp.getMixedHighPad()[reInd[0]];
+      if (reInd.size() == 1) {
+        collapsedPaddedShape[idx] = paddedType.getShape()[reInd[0]];
+        OpFoldResult paddedSize = affine::makeComposedFoldedAffineApply(
+            rewriter, loc, addMap, {l, h, expandedPaddedSizes[reInd[0]]});
+        expandedPaddedSizes[reInd[0]] = paddedSize;
+      }
+      newLow.push_back(l);
+      newHigh.push_back(h);
+    }
+
+    RankedTensorType collapsedPaddedType =
+        paddedType.clone(collapsedPaddedShape);
+    auto newPadOp = rewriter.create<tensor::PadOp>(
+        loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh,
+        padOp.getConstantPaddingValue(), padOp.getNofold());
+
+    rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
+        padOp, padOp.getResultType(), newPadOp.getResult(), reassociations,
+        expandedPaddedSizes);
+
+    return success();
+  }
+
+private:
+  ControlFusionFn controlFoldingReshapes;
+};
+
 /// Pattern to collapse dimensions.
 template <typename LinalgType>
 class CollapseLinalgDimensions : public OpRewritePattern<LinalgType> {
@@ -1937,6 +2079,8 @@ void mlir::linalg::populateFoldReshapeOpsByExpansionPatterns(
     const ControlFusionFn &controlFoldingReshapes) {
   patterns.add<FoldReshapeWithGenericOpByExpansion>(patterns.getContext(),
                                                     controlFoldingReshapes);
+  patterns.add<FoldPadWithProducerReshapeOpByExpansion>(patterns.getContext(),
+                                                        controlFoldingReshapes);
   patterns.add<FoldWithProducerReshapeOpByExpansion>(patterns.getContext(),
                                                      controlFoldingReshapes);
 }
@@ -1946,6 +2090,8 @@ void mlir::linalg::populateFoldReshapeOpsByCollapsingPatterns(
     const ControlFusionFn &controlFoldingReshapes) {
   patterns.add<FoldWithProducerReshapeOpByCollapsing>(patterns.getContext(),
                                                       controlFoldingReshapes);
+  patterns.add<FoldPadWithProducerReshapeOpByCollapsing>(
+      patterns.getContext(), controlFoldingReshapes);
 }
 
 void mlir::linalg::populateElementwiseOpsFusionPatterns(
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index b0a4de2da1e869314bee2fa4c5c5498da0cbb362..8b0e04fb61b1bd6f92aba15f0dd4f12d4b834f80 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -184,8 +184,7 @@ static void replaceIndexOpsByInductionVariables(RewriterBase &rewriter,
   for (Operation *loopOp : loopOps) {
     llvm::TypeSwitch<Operation *>(loopOp)
         .Case([&](scf::ParallelOp parallelOp) {
-          allIvs.append(parallelOp.getInductionVars().begin(),
-                        parallelOp.getInductionVars().end());
+          allIvs.append(parallelOp.getInductionVars());
         })
         .Case([&](scf::ForOp forOp) {
           allIvs.push_back(forOp.getInductionVar());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index fd314ef9f81347044e2fc88e35a3b2532b38042c..a0a0e11a6903d7a14fa211d9807b9bfd98cfe8f8 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -243,7 +243,7 @@ static void calculateTileOffsetsAndSizes(
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPointToStart(forallOp.getBody(0));
 
-  ValueRange threadIds = forallOp.getInductionVars();
+  SmallVector<Value> threadIds = forallOp.getInductionVars();
   SmallVector<OpFoldResult> nonZeroNumThreads =
       llvm::to_vector(llvm::make_filter_range(numThreads, [](OpFoldResult ofr) {
         return !isConstantIntValue(ofr, 0);
@@ -746,7 +746,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
                                            b.getIndexAttr(0));
       SmallVector<OpFoldResult> sizes = tiledSizes;
       sizes[reductionDim] = b.getIndexAttr(1);
-      outOffsets[reductionDim] = forallOp.getInductionVars().front();
+      outOffsets[reductionDim] = forallOp.getInductionVars()[0];
       // TODO: use SubsetExtractOpInterface once it is available.
       tiledDpsInitOperands.push_back(b.create<tensor::ExtractSliceOp>(
           loc, cast<RankedTensorType>(initOperand.getType()),
@@ -814,7 +814,7 @@ FailureOr<linalg::ForallReductionTilingResult> linalg::tileReductionUsingForall(
     int64_t sizeIdx = 0;
     for (int64_t i = 0, e = numThreads.size(); i < e; ++i) {
       if (i == reductionDim) {
-        resultOffsetsRank.push_back(forallOp.getInductionVars().front());
+        resultOffsetsRank.push_back(forallOp.getInductionVars()[0]);
         resultSizesRank.push_back(b.getIndexAttr(1));
         continue;
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 91dfac802ad67270140f5facd63928e97c895393..f30ef235e9cd345ad8a8ec1eff09b71f25997727 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -356,13 +356,6 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
 
 FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
                                                    tensor::UnPackOp unPackOp) {
-  // 1. Filter out NYI cases.
-  if (!unPackOp.getOuterDimsPerm().empty() &&
-      !isIdentityPermutation(unPackOp.getOuterDimsPerm())) {
-    return rewriter.notifyMatchFailure(unPackOp,
-                                       "non-identity outer dims perm NYI");
-  }
-
   Location loc = unPackOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(unPackOp);
@@ -391,36 +384,33 @@ FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
     return LowerUnPackOpResult{/*emptyOp=*/nullptr, /*transposeOp=*/nullptr,
                                /*reshapeOp=*/nullptr, extractSliceOp};
   }
-  // 2. Compute the permutation vector to move the last `numPackedDims` into
-  // the `innerPosDims` of a shape of rank `packedRank`.
-  int64_t numPackedDims = unPackOp.getInnerDimsPos().size();
-  auto lastDims = llvm::to_vector(
-      llvm::seq<int64_t>(packedRank - numPackedDims, packedRank));
-  PackingMetadata packingMetadata =
-      computePackingMetadata(packedRank, unPackOp.getInnerDimsPos());
-  SmallVector<int64_t> lastDimsToInsertPositionsPerm = computePermutationVector(
-      packedRank, lastDims, packingMetadata.insertPositions);
-
-  // 3. Compute the stripMinedShape: this is the packed shape without outer and
+
+  // 1. Compute the permutation vector to shuffle packed shape into the shape
+  // before any outer or inner permutations have been applied.
+  PackingMetadata packingMetadata;
+  SmallVector<int64_t> packedToStripMinedShapePerm =
+      tensor::getUnPackInverseSrcPerm(unPackOp, packingMetadata);
+
+  // 2. Compute the stripMinedShape: this is the packed shape without outer and
   // inner permutations.
   SmallVector<int64_t> stripMinedShape(packedTensorType.getShape());
-  applyPermutationToVector(stripMinedShape, lastDimsToInsertPositionsPerm);
+  applyPermutationToVector(stripMinedShape, packedToStripMinedShapePerm);
 
-  // 4. Transpose packedShape to stripMinedShape.
+  // 3. Transpose packedShape to stripMinedShape.
   RankedTensorType stripMinedTensorType =
       RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape);
   RankedTensorType collapsedType = tensor::CollapseShapeOp::inferCollapsedType(
       stripMinedTensorType, packingMetadata.reassociations);
 
-  // Get dynamic dims from input tensor based on lastDimsToInsertPositionsPerm
+  // Get dynamic dims from input tensor based on packedToStripMinedShapePerm
   // permutation.
   SmallVector<OpFoldResult, 4> dims =
       tensor::getMixedSizes(rewriter, loc, unPackOp.getSource());
-  applyPermutationToVector(dims, lastDimsToInsertPositionsPerm);
+  applyPermutationToVector(dims, packedToStripMinedShapePerm);
   auto emptyOp = rewriter.create<tensor::EmptyOp>(
       loc, dims, stripMinedTensorType.getElementType());
   auto transposeOp = rewriter.create<linalg::TransposeOp>(
-      loc, unPackOp.getSource(), emptyOp, lastDimsToInsertPositionsPerm);
+      loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
       DBGSNL(); DBGSNL(); llvm::interleaveComma(packingMetadata.insertPositions,
@@ -428,8 +418,8 @@ FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
       DBGSNL(); llvm::interleaveComma(packedTensorType.getShape(),
                                       DBGS() << "packedShape: ");
       DBGSNL();
-      llvm::interleaveComma(lastDimsToInsertPositionsPerm,
-                            DBGS() << "lastDimsToInsertPositionsPerm: ");
+      llvm::interleaveComma(packedToStripMinedShapePerm,
+                            DBGS() << "packedToStripMinedShapePerm: ");
       DBGSNL(); llvm::interleaveComma(
           packingMetadata.reassociations, DBGS() << "reassociations: ",
           [&](ReassociationIndices ri) {
@@ -439,12 +429,12 @@ FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
       llvm::interleaveComma(stripMinedShape, DBGS() << "stripMinedShape: ");
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
-  // 5. Collapse from the stripMinedShape to the padded result.
+  // 4. Collapse from the stripMinedShape to the padded result.
   auto reshapeOp = rewriter.create<tensor::CollapseShapeOp>(
       loc, collapsedType, transposeOp->getResult(0),
       packingMetadata.reassociations);
 
-  // 6. ExtractSlice.
+  // 5. ExtractSlice.
   int64_t destRank = destTensorType.getRank();
   auto extractSliceOp = rewriter.create<tensor::ExtractSliceOp>(
       loc, destTensorType, reshapeOp->getResult(0),
@@ -452,11 +442,11 @@ FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
       tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()),
       SmallVector<OpFoldResult>(destRank, one));
 
-  // 7. Inject a copy to preserve DPS.
+  // 6. Inject a copy to preserve DPS.
   auto copyOp = rewriter.create<linalg::CopyOp>(
       loc, extractSliceOp->getResult(0), unPackOp.getDest());
 
-  // 8. Replace unPackOp by extractSliceOp.
+  // 7. Replace unPackOp by copyOp.
   rewriter.replaceOp(unPackOp, copyOp->getResults());
 
   return LowerUnPackOpResult{emptyOp, transposeOp, reshapeOp, extractSliceOp};
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 107fd0690f193e548ca9b5c39a18a13a253a5dd9..5e94f4dc612a7b80c09375c0f60a8e7bbcf66e41 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -378,20 +378,20 @@ LogicalResult ForOp::verifyRegions() {
   return success();
 }
 
-std::optional<Value> ForOp::getSingleInductionVar() {
-  return getInductionVar();
+std::optional<SmallVector<Value>> ForOp::getLoopInductionVars() {
+  return SmallVector<Value>{getInductionVar()};
 }
 
-std::optional<OpFoldResult> ForOp::getSingleLowerBound() {
-  return OpFoldResult(getLowerBound());
+std::optional<SmallVector<OpFoldResult>> ForOp::getLoopLowerBounds() {
+  return SmallVector<OpFoldResult>{OpFoldResult(getLowerBound())};
 }
 
-std::optional<OpFoldResult> ForOp::getSingleStep() {
-  return OpFoldResult(getStep());
+std::optional<SmallVector<OpFoldResult>> ForOp::getLoopSteps() {
+  return SmallVector<OpFoldResult>{OpFoldResult(getStep())};
 }
 
-std::optional<OpFoldResult> ForOp::getSingleUpperBound() {
-  return OpFoldResult(getUpperBound());
+std::optional<SmallVector<OpFoldResult>> ForOp::getLoopUpperBounds() {
+  return SmallVector<OpFoldResult>{OpFoldResult(getUpperBound())};
 }
 
 std::optional<ResultRange> ForOp::getLoopResults() { return getResults(); }
@@ -1428,28 +1428,26 @@ SmallVector<Operation *> ForallOp::getCombiningOps(BlockArgument bbArg) {
   return storeOps;
 }
 
-std::optional<Value> ForallOp::getSingleInductionVar() {
-  if (getRank() != 1)
-    return std::nullopt;
-  return getInductionVar(0);
+std::optional<SmallVector<Value>> ForallOp::getLoopInductionVars() {
+  return SmallVector<Value>{getBody()->getArguments().take_front(getRank())};
 }
 
-std::optional<OpFoldResult> ForallOp::getSingleLowerBound() {
-  if (getRank() != 1)
-    return std::nullopt;
-  return getMixedLowerBound()[0];
+// Get lower bounds as OpFoldResult.
+std::optional<SmallVector<OpFoldResult>> ForallOp::getLoopLowerBounds() {
+  Builder b(getOperation()->getContext());
+  return getMixedValues(getStaticLowerBound(), getDynamicLowerBound(), b);
 }
 
-std::optional<OpFoldResult> ForallOp::getSingleUpperBound() {
-  if (getRank() != 1)
-    return std::nullopt;
-  return getMixedUpperBound()[0];
+// Get upper bounds as OpFoldResult.
+std::optional<SmallVector<OpFoldResult>> ForallOp::getLoopUpperBounds() {
+  Builder b(getOperation()->getContext());
+  return getMixedValues(getStaticUpperBound(), getDynamicUpperBound(), b);
 }
 
-std::optional<OpFoldResult> ForallOp::getSingleStep() {
-  if (getRank() != 1)
-    return std::nullopt;
-  return getMixedStep()[0];
+// Get steps as OpFoldResult.
+std::optional<SmallVector<OpFoldResult>> ForallOp::getLoopSteps() {
+  Builder b(getOperation()->getContext());
+  return getMixedValues(getStaticStep(), getDynamicStep(), b);
 }
 
 ForallOp mlir::scf::getForallOpThreadIndexOwner(Value val) {
@@ -3008,28 +3006,20 @@ void ParallelOp::print(OpAsmPrinter &p) {
 
 SmallVector<Region *> ParallelOp::getLoopRegions() { return {&getRegion()}; }
 
-std::optional<Value> ParallelOp::getSingleInductionVar() {
-  if (getNumLoops() != 1)
-    return std::nullopt;
-  return getBody()->getArgument(0);
+std::optional<SmallVector<Value>> ParallelOp::getLoopInductionVars() {
+  return SmallVector<Value>{getBody()->getArguments()};
 }
 
-std::optional<OpFoldResult> ParallelOp::getSingleLowerBound() {
-  if (getNumLoops() != 1)
-    return std::nullopt;
-  return getLowerBound()[0];
+std::optional<SmallVector<OpFoldResult>> ParallelOp::getLoopLowerBounds() {
+  return getLowerBound();
 }
 
-std::optional<OpFoldResult> ParallelOp::getSingleUpperBound() {
-  if (getNumLoops() != 1)
-    return std::nullopt;
-  return getUpperBound()[0];
+std::optional<SmallVector<OpFoldResult>> ParallelOp::getLoopUpperBounds() {
+  return getUpperBound();
 }
 
-std::optional<OpFoldResult> ParallelOp::getSingleStep() {
-  if (getNumLoops() != 1)
-    return std::nullopt;
-  return getStep()[0];
+std::optional<SmallVector<OpFoldResult>> ParallelOp::getLoopSteps() {
+  return getStep();
 }
 
 ParallelOp mlir::scf::getParallelForInductionVarOwner(Value val) {
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index 30699ecdde0a2024b279577bb52e27d65e5cd4d6..15b8aca5e267f70583edf9b178d80e6cc1c5176d 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -261,6 +261,8 @@ loopScheduling(scf::ForOp forOp,
     return 1;
   };
 
+  std::optional<int64_t> ubConstant = getConstantIntValue(forOp.getUpperBound());
+  std::optional<int64_t> lbConstant = getConstantIntValue(forOp.getLowerBound());
   DenseMap<Operation *, unsigned> opCycles;
   std::map<unsigned, std::vector<Operation *>> wrappedSchedule;
   for (Operation &op : forOp.getBody()->getOperations()) {
@@ -271,7 +273,14 @@ loopScheduling(scf::ForOp forOp,
       Operation *def = operand.getDefiningOp();
       if (!def)
         continue;
-      earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
+      if (ubConstant && lbConstant) {
+        unsigned ubInt = ubConstant.value();
+        unsigned lbInt = lbConstant.value();
+        auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def));
+        earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency);
+      } else {
+        earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def));
+      }
     }
     opCycles[&op] = earlyCycle;
     wrappedSchedule[earlyCycle % iterationInterval].push_back(&op);
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
index 198cb2e6cc69ef6e0499a7e4c31a56ec5514e3bd..5da1b76e929be49e6d6fa7053aa89bacee886c04 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
@@ -34,12 +34,9 @@ mlir::scf::forallToForLoop(RewriterBase &rewriter, scf::ForallOp forallOp,
   rewriter.setInsertionPoint(forallOp);
 
   Location loc = forallOp.getLoc();
-  SmallVector<Value> lbs = getValueOrCreateConstantIndexOp(
-      rewriter, loc, forallOp.getMixedLowerBound());
-  SmallVector<Value> ubs = getValueOrCreateConstantIndexOp(
-      rewriter, loc, forallOp.getMixedUpperBound());
-  SmallVector<Value> steps =
-      getValueOrCreateConstantIndexOp(rewriter, loc, forallOp.getMixedStep());
+  SmallVector<Value> lbs = forallOp.getLowerBound(rewriter);
+  SmallVector<Value> ubs = forallOp.getUpperBound(rewriter);
+  SmallVector<Value> steps = forallOp.getStep(rewriter);
   LoopNest loopNest = scf::buildLoopNest(rewriter, loc, lbs, ubs, steps);
 
   SmallVector<Value> ivs = llvm::map_to_vector(
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
index 1fc0331300379b5ad32146cda3fd015033309077..44e6840b03a3df173572797023054d917c6de472 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToParallel.cpp
@@ -35,12 +35,9 @@ LogicalResult mlir::scf::forallToParallelLoop(RewriterBase &rewriter,
         "only fully bufferized scf.forall ops can be lowered to scf.parallel");
 
   // Convert mixed bounds and steps to SSA values.
-  SmallVector<Value> lbs = getValueOrCreateConstantIndexOp(
-      rewriter, loc, forallOp.getMixedLowerBound());
-  SmallVector<Value> ubs = getValueOrCreateConstantIndexOp(
-      rewriter, loc, forallOp.getMixedUpperBound());
-  SmallVector<Value> steps =
-      getValueOrCreateConstantIndexOp(rewriter, loc, forallOp.getMixedStep());
+  SmallVector<Value> lbs = forallOp.getLowerBound(rewriter);
+  SmallVector<Value> ubs = forallOp.getUpperBound(rewriter);
+  SmallVector<Value> steps = forallOp.getStep(rewriter);
 
   // Create empty scf.parallel op.
   auto parallelOp = rewriter.create<scf::ParallelOp>(loc, lbs, ubs, steps);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 8545c7b9af8f73d405dda93765a3e1dcbe75a0c2..7fc29ec0139c20a90e816ac4308b212b5ae2c62a 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -4531,17 +4531,18 @@ struct FoldTensorCastProducerOp
     if (!hasTensorCastOperand)
       return failure();
 
-    SmallVector<Type, 4> newResultTypes;
-    newResultTypes.reserve(op->getNumResults());
+    SmallVector<Type, 4> newResultTypes(op->getResultTypes());
     SmallVector<Value, 4> newOperands;
     newOperands.reserve(op->getNumOperands());
+    // Assumes that the result has dpsInits followed by nonDpsInits.
+    int64_t dpsInitIdx = 0;
     for (OpOperand &opOperand : op->getOpOperands()) {
       auto tensorCastOp = opOperand.get().getDefiningOp<tensor::CastOp>();
       bool fold = canFoldIntoConsumerOp(tensorCastOp);
       newOperands.push_back(fold ? tensorCastOp.getOperand() : opOperand.get());
       if (op.isDpsInit(&opOperand) &&
           !llvm::isa<MemRefType>(newOperands.back().getType()))
-        newResultTypes.push_back(newOperands.back().getType());
+        newResultTypes[dpsInitIdx++] = newOperands.back().getType();
     }
 
     // Clone op.
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 5c6a32ce9a68d60edd812c1395d9341db710b91d..33016f84056e9b67027c592adc70578767977451 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -127,7 +127,20 @@ void transform::ApplyReassociativeReshapeFoldingPatternsOp::populatePatterns(
 
 void transform::ApplyRewriteTensorOpsAsConstantPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
-  tensor::populateRewriteAsConstantPatterns(patterns);
+  ControlFoldFn defaultControlFn = [](OpOperand *fusedOperand) {
+    Operation *producer = fusedOperand->get().getDefiningOp();
+    return producer && producer->hasOneUse();
+  };
+
+  ControlFoldFn aggressiveControlFn = [](OpOperand *fusedOperand) {
+    return true;
+  };
+
+  // Add folding with reshape by expansion patterns.
+  if (getAggressive())
+    tensor::populateRewriteAsConstantPatterns(patterns, aggressiveControlFn);
+  else
+    tensor::populateRewriteAsConstantPatterns(patterns, defaultControlFn);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp
index 5d6e3ec9756af2f5313f638be50dbb7df30c90d6..c681cadcb27cb252b8b7ce52486ceaa7f94aaf77 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp
@@ -48,6 +48,34 @@ static LogicalResult isPackOn1D(RewriterBase &rewriter, Operation *op,
   return success();
 }
 
+// If the `linalgOp` represents a transpose, return the permutation vector for
+// the transpose. Otherwise, return failure.
+static FailureOr<SmallVector<int64_t>>
+getTransposeOpPermutation(linalg::LinalgOp linalgOp) {
+  if (auto transposeOp = dyn_cast<linalg::TransposeOp>(linalgOp.getOperation()))
+    return SmallVector<int64_t>(transposeOp.getPermutation());
+  if (linalgOp.getNumParallelLoops() != linalgOp.getNumLoops())
+    return failure();
+
+  if (linalgOp.getNumDpsInputs() != 1 || linalgOp.getNumDpsInits() != 1)
+    return failure();
+  auto mapRange = linalgOp.getIndexingMapsArray();
+  if (!mapRange.front().isPermutation() || !mapRange.back().isPermutation() ||
+      mapRange.front() == mapRange.back()) {
+    return failure();
+  }
+  if (!llvm::hasSingleElement(linalgOp.getBlock()->getOperations()))
+    return failure();
+  AffineMap outMap = mapRange.back();
+  AffineMap inMap = mapRange.front();
+  // To get the permutation, look at each output index and find which
+  // dimension in the input we're reading from for that index.
+  return llvm::map_to_vector(outMap.getResults(),
+                             [&](AffineExpr expr) -> int64_t {
+                               return *inMap.getResultPosition(expr);
+                             });
+}
+
 /// Packing one-dimensional tensor can be expressed as an expand shape op.
 struct SimplifyPackToExpandShape : public OpRewritePattern<PackOp> {
   using OpRewritePattern<PackOp>::OpRewritePattern;
@@ -246,14 +274,10 @@ static bool checkAndPermute(ArrayRef<int64_t> permutation,
 
   for (unsigned int i = 0; i < rank; ++i) {
     int64_t remappedPosition = permutation[i];
-
-    if (!inVec.empty()) {
-      if (remappedPosition >= rank) {
-        return false;
-      }
+    if (remappedPosition >= rank)
+      return false;
+    if (!inVec.empty())
       remappedPosition = inVec[remappedPosition];
-    }
-
     resVec.push_back(remappedPosition);
   }
 
@@ -263,20 +287,25 @@ static bool checkAndPermute(ArrayRef<int64_t> permutation,
 /// Fold 'pack' -> 'transpose' into 'pack' since 'pack' already has transpose
 /// semantics.
 struct FoldProducerPackWithConsumerLinalgTransposeOp
-    : public OpRewritePattern<linalg::TransposeOp> {
-  using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
+    : public OpInterfaceRewritePattern<linalg::LinalgOp> {
+  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
+  LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
-    auto packOp = transposeOp.getOperand(0).getDefiningOp<PackOp>();
+    auto packOp = linalgOp->getOperand(0).getDefiningOp<PackOp>();
 
     if (!packOp)
       return failure();
 
+    FailureOr<SmallVector<int64_t>> maybePerm =
+        getTransposeOpPermutation(linalgOp);
+    if (failed(maybePerm))
+      return failure();
+
     auto innerDimsPos = packOp.getInnerDimsPos();
     auto mixedInnerTiles = packOp.getMixedTiles();
     auto outerDimsPerm = packOp.getOuterDimsPerm();
-    auto transposePerm = transposeOp.getPermutation();
+    auto transposePerm = maybePerm.value();
     SmallVector<int64_t> newOuterDimsPermVec;
     SmallVector<int64_t> newInnerDimsPosVec;
     SmallVector<OpFoldResult> newMixedInnerTilesVec;
@@ -285,7 +314,7 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
     if (!checkAndPermute(transposePerm, outerDimsPerm, newOuterDimsPermVec,
                          srcRank))
       return rewriter.notifyMatchFailure(
-          transposeOp,
+          linalgOp,
           "Cannot fold in tensor.pack if a tile dimension was transposed "
           "with a non-tile dimension in linalg.transpose.");
 
@@ -297,11 +326,11 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
     }
 
     Value output = packOp.createDestinationTensor(
-        rewriter, transposeOp.getLoc(), packOp.getSource(),
-        newMixedInnerTilesVec, newInnerDimsPosVec, newOuterDimsPermVec);
+        rewriter, linalgOp.getLoc(), packOp.getSource(), newMixedInnerTilesVec,
+        newInnerDimsPosVec, newOuterDimsPermVec);
 
     rewriter.replaceOpWithNewOp<PackOp>(
-        transposeOp, packOp.getSource(), output, newInnerDimsPosVec,
+        linalgOp, packOp.getSource(), output, newInnerDimsPosVec,
         newMixedInnerTilesVec, packOp.getPaddingValue(), newOuterDimsPermVec);
 
     return success();
@@ -316,12 +345,16 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
 
   LogicalResult matchAndRewrite(PackOp packOp,
                                 PatternRewriter &rewriter) const override {
-    auto transposeOp = packOp.getSource().getDefiningOp<linalg::TransposeOp>();
+    auto linalgOp = packOp.getSource().getDefiningOp<linalg::LinalgOp>();
+    if (!linalgOp)
+      return failure();
 
-    if (!transposeOp)
+    FailureOr<SmallVector<int64_t>> maybePerm =
+        getTransposeOpPermutation(linalgOp);
+    if (failed(maybePerm))
       return failure();
 
-    auto transposePermutation = transposeOp.getPermutation();
+    auto transposePermutation = maybePerm.value();
     auto outerDimsPerm = packOp.getOuterDimsPerm();
     auto innerDimsPos = packOp.getInnerDimsPos();
     SmallVector<int64_t> newInnerDimsPosVec;
@@ -337,11 +370,11 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
       newInnerDimsPosVec.push_back(transposePermutation[dim]);
 
     Value output = packOp.createDestinationTensor(
-        rewriter, packOp.getLoc(), transposeOp.getOperand(0),
+        rewriter, packOp.getLoc(), linalgOp->getOperand(0),
         packOp.getMixedTiles(), newInnerDimsPosVec, newOuterDimsPermVec);
 
     rewriter.replaceOpWithNewOp<PackOp>(
-        packOp, transposeOp.getOperand(0), output, newInnerDimsPosVec,
+        packOp, linalgOp->getOperand(0), output, newInnerDimsPosVec,
         packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPermVec);
 
     return success();
@@ -351,34 +384,38 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
 /// Fold 'unpack' -> 'transpose' into 'unpack' since 'unpack' already has
 /// transpose semantics.
 struct FoldProducerUnPackWithConsumerLinalgTransposeOp
-    : public OpRewritePattern<linalg::TransposeOp> {
-  using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
+    : public OpInterfaceRewritePattern<linalg::LinalgOp> {
+  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
+  LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
-    auto unPackOp = transposeOp.getOperand(0).getDefiningOp<UnPackOp>();
+    auto unPackOp = linalgOp->getOperand(0).getDefiningOp<UnPackOp>();
 
     if (!unPackOp)
       return failure();
 
-    auto transposePermutation = transposeOp.getPermutation();
+    FailureOr<SmallVector<int64_t>> maybePerm =
+        getTransposeOpPermutation(linalgOp);
+    if (failed(maybePerm))
+      return failure();
+
     auto outerDimsPerm = unPackOp.getOuterDimsPerm();
     auto innerDimsPos = unPackOp.getInnerDimsPos();
     SmallVector<int64_t> newInnerDimsPosVec;
     SmallVector<int64_t> newOuterDimsPermVec =
-        llvm::to_vector(transposePermutation);
-
-    if (!outerDimsPerm.empty())
-      applyPermutationToVector(newOuterDimsPermVec, outerDimsPerm);
+        invertPermutationVector(maybePerm.value());
 
     // Can't use applyPermutationToVector for newInnerDimsPosVec since input and
     // permutation rank won't necessarily be equal in all cases.
     for (auto dim : innerDimsPos)
-      newInnerDimsPosVec.push_back(transposePermutation[dim]);
+      newInnerDimsPosVec.push_back(newOuterDimsPermVec[dim]);
+
+    if (!outerDimsPerm.empty())
+      applyPermutationToVector(newOuterDimsPermVec, outerDimsPerm);
 
     // Reuse the destination of the transpose op.
     rewriter.replaceOpWithNewOp<UnPackOp>(
-        transposeOp, unPackOp.getSource(), transposeOp.getDpsInits()[0],
+        linalgOp, unPackOp.getSource(), linalgOp.getDpsInits()[0],
         newInnerDimsPosVec, unPackOp.getMixedTiles(), newOuterDimsPermVec);
 
     return success();
@@ -393,13 +430,17 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
   LogicalResult matchAndRewrite(UnPackOp unPackOp,
                                 PatternRewriter &rewriter) const override {
-    auto transposeOp =
-        unPackOp.getSource().getDefiningOp<linalg::TransposeOp>();
+    auto linalgOp = unPackOp.getSource().getDefiningOp<linalg::LinalgOp>();
+    if (!linalgOp)
+      return failure();
 
-    if (!transposeOp)
+    FailureOr<SmallVector<int64_t>> maybePerm =
+        getTransposeOpPermutation(linalgOp);
+    if (failed(maybePerm))
       return failure();
 
-    auto transposePermutation = transposeOp.getPermutation();
+    SmallVector<int64_t> inverseTransposePerm =
+        invertPermutationVector(maybePerm.value());
     auto outerDimsPerm = unPackOp.getOuterDimsPerm();
     auto innerDimsPos = unPackOp.getInnerDimsPos();
     int64_t destRank = unPackOp.getSourceRank() - innerDimsPos.size();
@@ -408,7 +449,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
     SmallVector<int64_t> newInnerDimsPosVec;
     SmallVector<OpFoldResult> newMixedInnerTilesVec;
 
-    if (!checkAndPermute(transposePermutation, outerDimsPerm,
+    if (!checkAndPermute(inverseTransposePerm, outerDimsPerm,
                          newOuterDimsPermVec, destRank))
       return rewriter.notifyMatchFailure(
           unPackOp,
@@ -416,18 +457,18 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
           "with a non-tile dimension in linalg.transpose.");
 
     // Process transpose operation for tiled inner dimensions
-    for (unsigned int i = destRank; i < transposePermutation.size(); ++i) {
-      int64_t remappedPosition = transposePermutation[i] - destRank;
+    for (unsigned int i = destRank; i < inverseTransposePerm.size(); ++i) {
+      int64_t remappedPosition = inverseTransposePerm[i] - destRank;
       newMixedInnerTilesVec.push_back(mixedInnerTilesVec[remappedPosition]);
       newInnerDimsPosVec.push_back(innerDimsPos[remappedPosition]);
     }
 
     Value output = unPackOp.createDestinationTensor(
-        rewriter, unPackOp.getLoc(), transposeOp.getOperand(0),
+        rewriter, unPackOp.getLoc(), linalgOp->getOperand(0),
         newMixedInnerTilesVec, newInnerDimsPosVec, newOuterDimsPermVec);
 
     rewriter.replaceOpWithNewOp<UnPackOp>(
-        unPackOp, transposeOp.getOperand(0), output, newInnerDimsPosVec,
+        unPackOp, linalgOp->getOperand(0), output, newInnerDimsPosVec,
         newMixedInnerTilesVec, newOuterDimsPermVec);
 
     return success();
diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
index 11e1de543ac91e782b6bc8ab1141aad5cbbdff1d..7c9fced540adb9417795e503715243d94e2e84cd 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp
@@ -8,9 +8,12 @@
 //
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 
+#include "llvm/ADT/TypeSwitch.h"
+
 using namespace mlir;
 using namespace mlir::tensor;
 
@@ -45,9 +48,169 @@ struct GenerateToConstant : public OpRewritePattern<GenerateOp> {
   }
 };
 
+/// Transform a linear index from one indexing space to another given:
+///
+/// - the shape of the source indexing space,
+/// - the strides of the target indexing space,
+/// - a linear index into the source indexing space.
+///
+/// This function is logically a sequence of linearize/delinearize over
+/// different bases but avoids allocating intermediate SmallVectors.
+int64_t transformIndexSpace(ArrayRef<int64_t> inputShape,
+                            ArrayRef<int64_t> outputStrides,
+                            int64_t srcLinearIndex) {
+  assert(inputShape.size() == outputStrides.size());
+
+  int64_t dstLinearIndex = 0;
+
+  for (int64_t dim = inputShape.size() - 1; dim >= 0; --dim) {
+    // Compute the index into the current dimension of the source tensor.
+    // `quotient` is the remaining linear index after accounting for the
+    // current dimension.
+    //
+    // `remainder` is the index into the source tensor for the current
+    // dimension.
+    auto [quotient, remainder] = std::div(srcLinearIndex, inputShape[dim]);
+
+    srcLinearIndex = quotient;
+
+    // Add the contribution of the current dimension to the output using the
+    // permutation map.
+    dstLinearIndex += outputStrides[dim] * remainder;
+  }
+
+  return dstLinearIndex;
+}
+
+template <typename ElemType, typename AttrType>
+Value constantFoldPadOp(PatternRewriter &rewriter, Location loc,
+                        DenseElementsAttr input, AttrType padValue,
+                        ArrayRef<int64_t> padLow, ArrayRef<int64_t> padHigh) {
+  auto inputValues = input.tryGetValues<ElemType>();
+  if (failed(inputValues))
+    return nullptr;
+
+  auto oldShape = input.getType().getShape();
+
+  // Compute the output shape of the new value.
+  auto newShape =
+      llvm::map_to_vector(llvm::zip(oldShape, padLow, padHigh),
+                          [](std::tuple<int64_t, int64_t, int64_t> pack) {
+                            auto [old, low, high] = pack;
+                            return old + low + high;
+                          });
+
+  int64_t outputSize = computeProduct(newShape);
+
+  // Fully initialize the vector with the padding value.
+  // The non-padded area will then be copied.
+  SmallVector<ElemType> values(outputSize, padValue.getValue());
+
+  // Strides for input and output are used to transform between the indexing
+  // space of the input and output tensors.
+  SmallVector<int64_t> outputStrides = computeStrides(newShape);
+
+  // The contribution of the low padding to the offset in the output tensor.
+  // This is the starting position of the source tensor within the padding
+  // tensor.
+  int64_t startingOffset = linearize(padLow, outputStrides);
+
+  // Copy values from the input tensor to the corresponding sub-region
+  // of the output tensor.
+  for (auto [inputIndex, inputValue] : llvm::enumerate(*inputValues)) {
+    auto outputIndex = transformIndexSpace(oldShape, outputStrides, inputIndex);
+    values[outputIndex + startingOffset] = inputValue;
+  }
+
+  // Create an attribute for the folded value.
+  auto newType = input.getType().clone(newShape);
+  auto newAttr = DenseElementsAttr::get(newType, values);
+
+  Operation *constantOp =
+      rewriter.getContext()
+          ->getLoadedDialect<TensorDialect>()
+          ->materializeConstant(rewriter, newAttr, newType, loc);
+
+  return constantOp ? constantOp->getResult(0) : nullptr;
+}
+
+struct PadOpToConstant final : public OpRewritePattern<PadOp> {
+
+  PadOpToConstant(MLIRContext *context, const ControlFoldFn &controlFn,
+                  PatternBenefit benefit = 1)
+      : OpRewritePattern<PadOp>(context, benefit), controlFn{controlFn} {}
+
+  LogicalResult matchAndRewrite(PadOp padTensorOp,
+                                PatternRewriter &rewriter) const override {
+    if (padTensorOp.getNofold())
+      return rewriter.notifyMatchFailure(
+          padTensorOp, "refusing to fold nofold pad operation");
+
+    TypedValue<RankedTensorType> input = padTensorOp.getSource();
+    RankedTensorType resultType = padTensorOp.getResult().getType();
+
+    DenseElementsAttr inputAttr = nullptr;
+    if (!matchPattern(input, m_Constant(&inputAttr)))
+      return failure();
+
+    Value paddingValue = padTensorOp.getConstantPaddingValue();
+
+    // Extract the constant value used for padding or bail out.
+    Attribute paddingAttr = nullptr;
+    if (!paddingValue || !matchPattern(paddingValue, m_Constant(&paddingAttr)))
+      return rewriter.notifyMatchFailure(padTensorOp,
+                                         "unable to get constant value");
+
+    // Try to extract the constant values of the low and high padding.
+    auto lowPad = getConstantIntValues(padTensorOp.getMixedLowPad());
+    auto highPad = getConstantIntValues(padTensorOp.getMixedHighPad());
+
+    // If the padding cannot be extracted, bail out.
+    if (!lowPad || !highPad)
+      return rewriter.notifyMatchFailure(padTensorOp,
+                                         "unable to extract constant padding");
+
+    // We have a potential candidate, consult the control function to
+    // determine if the op should fold.
+    if (!controlFn(&padTensorOp.getSourceMutable()))
+      return rewriter.notifyMatchFailure(padTensorOp,
+                                         "not folding due to cost function");
+
+    Location loc = padTensorOp.getLoc();
+
+    // Try constant folding the supported cases of integer and float values.
+    Value newOp =
+        llvm::TypeSwitch<Attribute, Value>(paddingAttr)
+            .Case([&](FloatAttr floatAttr) {
+              return constantFoldPadOp<llvm::APFloat>(
+                  rewriter, loc, inputAttr, floatAttr, *lowPad, *highPad);
+            })
+            .Case([&](IntegerAttr integerAttr) {
+              return constantFoldPadOp<llvm::APInt>(
+                  rewriter, loc, inputAttr, integerAttr, *lowPad, *highPad);
+            })
+            .Default(Value());
+
+    if (!newOp)
+      return rewriter.notifyMatchFailure(padTensorOp,
+                                         "tensor type not supported");
+
+    if (newOp.getType() != resultType)
+      newOp = rewriter.create<tensor::CastOp>(loc, resultType, newOp);
+
+    rewriter.replaceOp(padTensorOp, newOp);
+    return success();
+  }
+
+private:
+  ControlFoldFn controlFn;
+};
+
 } // namespace
 
 void mlir::tensor::populateRewriteAsConstantPatterns(
-    RewritePatternSet &patterns) {
+    RewritePatternSet &patterns, const ControlFoldFn &controlFn) {
   patterns.add<GenerateToConstant>(patterns.getContext());
+
+  patterns.add<PadOpToConstant>(patterns.getContext(), controlFn);
 }
diff --git a/mlir/lib/Dialect/Utils/IndexingUtils.cpp b/mlir/lib/Dialect/Utils/IndexingUtils.cpp
index 4c960659d80cb7b779109d30492ae88eef1a1f22..aba225be720c31c70d2b13e6a960b228eb0ebed3 100644
--- a/mlir/lib/Dialect/Utils/IndexingUtils.cpp
+++ b/mlir/lib/Dialect/Utils/IndexingUtils.cpp
@@ -92,7 +92,7 @@ int64_t mlir::computeProduct(ArrayRef<int64_t> basis) {
   assert(llvm::all_of(basis, [](int64_t s) { return s > 0; }) &&
          "basis must be nonnegative");
   if (basis.empty())
-    return 0;
+    return 1;
   return std::accumulate(basis.begin(), basis.end(), 1,
                          std::multiplies<int64_t>());
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index 77c97b2f1497c443c319b1d850cc9ee66ad2b61b..f7e01c7b12e4fdfab53e3af96694f223e4910c2f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -79,6 +79,73 @@ private:
   int64_t targetRank = 1;
 };
 
+/// A one-shot unrolling of vector.deinterleave to the `targetRank`.
+///
+/// Example:
+///
+/// ```mlir
+/// %0, %1 = vector.deinterleave %a : vector<1x2x3x8xi64> -> vector<1x2x3x4xi64>
+/// ```
+/// Would be unrolled to:
+/// ```mlir
+/// %result = arith.constant dense<0> : vector<1x2x3x4xi64>
+/// %0 = vector.extract %a[0, 0, 0]                  ─┐
+///        : vector<8xi64> from vector<1x2x3x8xi64>   |
+/// %1, %2 = vector.deinterleave %0                   |
+///        : vector<8xi64> -> vector<4xi64>           | -- Initial deinterleave
+/// %3 = vector.insert %1, %result [0, 0, 0]          |    operation unrolled.
+///        : vector<4xi64> into vector<1x2x3x4xi64>   |
+/// %4 = vector.insert %2, %result [0, 0, 0]          |
+///        : vector<4xi64> into vector<1x2x3x4xi64>   ┘
+/// %5 = vector.extract %a[0, 0, 1]                  ─┐
+///        : vector<8xi64> from vector<1x2x3x8xi64>   |
+/// %6, %7 = vector.deinterleave %5                   |
+///        : vector<8xi64> -> vector<4xi64>           | -- Recursive pattern for
+/// %8 = vector.insert %6, %3 [0, 0, 1]               |    subsequent unrolled
+///        : vector<4xi64> into vector<1x2x3x4xi64>   |    deinterleave
+/// %9 = vector.insert %7, %4 [0, 0, 1]               |    operations. Repeated
+///        : vector<4xi64> into vector<1x2x3x4xi64>   ┘    5x in this case.
+/// ```
+///
+/// Note: If any leading dimension before the `targetRank` is scalable the
+/// unrolling will stop before the scalable dimension.
+class UnrollDeinterleaveOp final
+    : public OpRewritePattern<vector::DeinterleaveOp> {
+public:
+  UnrollDeinterleaveOp(int64_t targetRank, MLIRContext *context,
+                       PatternBenefit benefit = 1)
+      : OpRewritePattern(context, benefit), targetRank(targetRank) {};
+
+  LogicalResult matchAndRewrite(vector::DeinterleaveOp op,
+                                PatternRewriter &rewriter) const override {
+    VectorType resultType = op.getResultVectorType();
+    auto unrollIterator = vector::createUnrollIterator(resultType, targetRank);
+    if (!unrollIterator)
+      return failure();
+
+    auto loc = op.getLoc();
+    Value emptyResult = rewriter.create<arith::ConstantOp>(
+        loc, resultType, rewriter.getZeroAttr(resultType));
+    Value evenResult = emptyResult;
+    Value oddResult = emptyResult;
+
+    for (auto position : *unrollIterator) {
+      auto extractSrc =
+          rewriter.create<vector::ExtractOp>(loc, op.getSource(), position);
+      auto deinterleave =
+          rewriter.create<vector::DeinterleaveOp>(loc, extractSrc);
+      evenResult = rewriter.create<vector::InsertOp>(
+          loc, deinterleave.getRes1(), evenResult, position);
+      oddResult = rewriter.create<vector::InsertOp>(loc, deinterleave.getRes2(),
+                                                    oddResult, position);
+    }
+    rewriter.replaceOp(op, ValueRange{evenResult, oddResult});
+    return success();
+  }
+
+private:
+  int64_t targetRank = 1;
+};
 /// Rewrite vector.interleave op into an equivalent vector.shuffle op, when
 /// applicable: `sourceType` must be 1D and non-scalable.
 ///
@@ -116,7 +183,8 @@ struct InterleaveToShuffle final : OpRewritePattern<vector::InterleaveOp> {
 
 void mlir::vector::populateVectorInterleaveLoweringPatterns(
     RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) {
-  patterns.add<UnrollInterleaveOp>(targetRank, patterns.getContext(), benefit);
+  patterns.add<UnrollInterleaveOp, UnrollDeinterleaveOp>(
+      targetRank, patterns.getContext(), benefit);
 }
 
 void mlir::vector::populateVectorInterleaveToShufflePatterns(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 997b56a1ce14262ae0a5d0daaa85e2bf65d17e4b..c131fde517f8055a594d8749908f3ae35641a274 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -505,25 +505,61 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc,
   return rewriter.create<memref::CollapseShapeOp>(loc, input, reassociation);
 }
 
-/// Checks that the indices corresponding to dimensions starting at
-/// `firstDimToCollapse` are constant 0, and writes to `outIndices`
-/// the truncated indices where `firstDimToCollapse` is now the innermost dim.
-/// TODO: Extract the logic that writes to outIndices so that this method
-/// simply checks one pre-condition.
-static LogicalResult
-checkAndCollapseInnerZeroIndices(ValueRange indices, int64_t firstDimToCollapse,
-                                 SmallVector<Value> &outIndices) {
-  int64_t rank = indices.size();
-  if (firstDimToCollapse >= rank)
-    return failure();
-  for (int64_t i = firstDimToCollapse; i < rank; ++i) {
-    std::optional<int64_t> cst = getConstantIntValue(indices[i]);
-    if (!cst || cst.value() != 0)
-      return failure();
+/// Returns the new indices that collapses the inner dimensions starting from
+/// the `firstDimToCollapse` dimension.
+static SmallVector<Value> getCollapsedIndices(RewriterBase &rewriter,
+                                              Location loc,
+                                              ArrayRef<int64_t> shape,
+                                              ValueRange indices,
+                                              int64_t firstDimToCollapse) {
+  assert(firstDimToCollapse < static_cast<int64_t>(indices.size()));
+
+  // If all the collapsed indices are zero then no extra logic is needed.
+  // Otherwise, a new offset/index has to be computed.
+  SmallVector<Value> indicesAfterCollapsing(
+      indices.begin(), indices.begin() + firstDimToCollapse);
+  SmallVector<Value> indicesToCollapse(indices.begin() + firstDimToCollapse,
+                                       indices.end());
+  if (llvm::all_of(indicesToCollapse, isZeroIndex)) {
+    indicesAfterCollapsing.push_back(indicesToCollapse[0]);
+    return indicesAfterCollapsing;
+  }
+
+  // Compute the remaining trailing index/offset required for reading from
+  // the collapsed memref:
+  //
+  //    offset = 0
+  //    for (i = firstDimToCollapse; i < outputRank; ++i)
+  //      offset += sourceType.getDimSize(i) * transferReadOp.indices[i]
+  //
+  // For this example:
+  //   %2 = vector.transfer_read/write %arg4[%c0, %arg0, %c0] (...) :
+  //      memref<1x43x2xi32>, vector<1x2xi32>
+  // which would be collapsed to:
+  //   %1 = vector.transfer_read/write %collapse_shape[%c0, %offset] (...) :
+  //      memref<1x86xi32>, vector<2xi32>
+  // one would get the following offset:
+  //    %offset = %arg0 * 43
+  OpFoldResult collapsedOffset =
+      rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
+
+  auto collapsedStrides = computeSuffixProduct(
+      ArrayRef<int64_t>(shape.begin() + firstDimToCollapse, shape.end()));
+
+  // Compute the collapsed offset.
+  auto &&[collapsedExpr, collapsedVals] =
+      computeLinearIndex(collapsedOffset, collapsedStrides, indicesToCollapse);
+  collapsedOffset = affine::makeComposedFoldedAffineApply(
+      rewriter, loc, collapsedExpr, collapsedVals);
+
+  if (collapsedOffset.is<Value>()) {
+    indicesAfterCollapsing.push_back(collapsedOffset.get<Value>());
+  } else {
+    indicesAfterCollapsing.push_back(rewriter.create<arith::ConstantIndexOp>(
+        loc, *getConstantIntValue(collapsedOffset)));
   }
-  outIndices = indices;
-  outIndices.resize(firstDimToCollapse + 1);
-  return success();
+
+  return indicesAfterCollapsing;
 }
 
 namespace {
@@ -594,54 +630,9 @@ public:
         AffineMap::get(collapsedRank, 0, dimExprs, rewriter.getContext());
 
     // 2.2 New indices
-    // If all the collapsed indices are zero then no extra logic is needed.
-    // Otherwise, a new offset/index has to be computed.
-    SmallVector<Value> collapsedIndices;
-    if (failed(checkAndCollapseInnerZeroIndices(transferReadOp.getIndices(),
-                                                firstDimToCollapse,
-                                                collapsedIndices))) {
-      // Copy all the leading indices.
-      SmallVector<Value> indices = transferReadOp.getIndices();
-      collapsedIndices.append(indices.begin(),
-                              indices.begin() + firstDimToCollapse);
-
-      // Compute the remaining trailing index/offset required for reading from
-      // the collapsed memref:
-      //
-      //    offset = 0
-      //    for (i = firstDimToCollapse; i < outputRank; ++i)
-      //      offset += sourceType.getDimSize(i) * transferReadOp.indices[i]
-      //
-      // For this example:
-      //   %2 = vector.transfer_read %arg4[%c0, %arg0, %c0] (...) :
-      //      memref<1x43x2xi32>, vector<1x2xi32>
-      // which would be collapsed to:
-      //   %1 = vector.transfer_read %collapse_shape[%c0, %offset] (...) :
-      //      memref<1x86xi32>, vector<2xi32>
-      // one would get the following offset:
-      //    %offset = %arg0 * 43
-      OpFoldResult collapsedOffset =
-          rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult();
-
-      auto sourceShape = sourceType.getShape();
-      auto collapsedStrides = computeSuffixProduct(ArrayRef<int64_t>(
-          sourceShape.begin() + firstDimToCollapse, sourceShape.end()));
-
-      // Compute the collapsed offset.
-      ArrayRef<Value> indicesToCollapse(indices.begin() + firstDimToCollapse,
-                                        indices.end());
-      auto &&[collapsedExpr, collapsedVals] = computeLinearIndex(
-          collapsedOffset, collapsedStrides, indicesToCollapse);
-      collapsedOffset = affine::makeComposedFoldedAffineApply(
-          rewriter, loc, collapsedExpr, collapsedVals);
-
-      if (collapsedOffset.is<Value>()) {
-        collapsedIndices.push_back(collapsedOffset.get<Value>());
-      } else {
-        collapsedIndices.push_back(rewriter.create<arith::ConstantIndexOp>(
-            loc, *getConstantIntValue(collapsedOffset)));
-      }
-    }
+    SmallVector<Value> collapsedIndices =
+        getCollapsedIndices(rewriter, loc, sourceType.getShape(),
+                            transferReadOp.getIndices(), firstDimToCollapse);
 
     // 3. Create new vector.transfer_read that reads from the collapsed memref
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
@@ -697,8 +688,7 @@ public:
       return failure();
     if (!vector::isContiguousSlice(sourceType, vectorType))
       return failure();
-    int64_t firstContiguousInnerDim =
-        sourceType.getRank() - vectorType.getRank();
+    int64_t firstDimToCollapse = sourceType.getRank() - vectorType.getRank();
     // TODO: generalize this pattern, relax the requirements here.
     if (transferWriteOp.hasOutOfBoundsDim())
       return failure();
@@ -706,22 +696,23 @@ public:
       return failure();
     if (transferWriteOp.getMask())
       return failure();
-    SmallVector<Value> collapsedIndices;
-    if (failed(checkAndCollapseInnerZeroIndices(transferWriteOp.getIndices(),
-                                                firstContiguousInnerDim,
-                                                collapsedIndices)))
-      return failure();
+
+    SmallVector<Value> collapsedIndices =
+        getCollapsedIndices(rewriter, loc, sourceType.getShape(),
+                            transferWriteOp.getIndices(), firstDimToCollapse);
 
     Value collapsedSource =
-        collapseInnerDims(rewriter, loc, source, firstContiguousInnerDim);
+        collapseInnerDims(rewriter, loc, source, firstDimToCollapse);
     MemRefType collapsedSourceType =
         cast<MemRefType>(collapsedSource.getType());
     int64_t collapsedRank = collapsedSourceType.getRank();
-    assert(collapsedRank == firstContiguousInnerDim + 1);
+    assert(collapsedRank == firstDimToCollapse + 1);
+
     SmallVector<AffineExpr, 1> dimExprs{
-        getAffineDimExpr(firstContiguousInnerDim, rewriter.getContext())};
+        getAffineDimExpr(firstDimToCollapse, rewriter.getContext())};
     auto collapsedMap =
         AffineMap::get(collapsedRank, 0, dimExprs, rewriter.getContext());
+
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     Value flatVector =
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 22959224d56c2f30fa7623d692f2d202b17afee1..5ef47fbbe1ce09cd1be3089448a5a3c83155807f 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -110,7 +110,7 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
 
   dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
   dispatchIndexOpFoldResults(shape, dynamicShape, staticShape);
-  dispatchIndexOpFoldResults(strides, dynamicStrides, staticOffsets);
+  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
 
   auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
   auto staticShapeAttr = builder.getDenseI64ArrayAttr(staticShape);
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index 29e36210f1270919c5049ed2b4a6066af9510141..6a362afc52f25913cccb5e67c355850b6f6b8504 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -2061,7 +2061,8 @@ void AsmPrinter::Impl::printLocationInternal(LocationAttr loc, bool pretty,
 
 /// Print a floating point value in a way that the parser will be able to
 /// round-trip losslessly.
-static void printFloatValue(const APFloat &apValue, raw_ostream &os) {
+static void printFloatValue(const APFloat &apValue, raw_ostream &os,
+                            bool *printedHex = nullptr) {
   // We would like to output the FP constant value in exponential notation,
   // but we cannot do this if doing so will lose precision.  Check here to
   // make sure that we only output it in exponential format if we can parse
@@ -2102,6 +2103,8 @@ static void printFloatValue(const APFloat &apValue, raw_ostream &os) {
 
   // Print special values in hexadecimal format. The sign bit should be included
   // in the literal.
+  if (printedHex)
+    *printedHex = true;
   SmallVector<char, 16> str;
   APInt apInt = apValue.bitcastToAPInt();
   apInt.toString(str, /*Radix=*/16, /*Signed=*/false,
@@ -2275,10 +2278,12 @@ void AsmPrinter::Impl::printAttributeImpl(Attribute attr,
       return;
 
   } else if (auto floatAttr = llvm::dyn_cast<FloatAttr>(attr)) {
-    printFloatValue(floatAttr.getValue(), os);
+    bool printedHex = false;
+    printFloatValue(floatAttr.getValue(), os, &printedHex);
 
     // FloatAttr elides the type if F64.
-    if (typeElision == AttrTypeElision::May && floatAttr.getType().isF64())
+    if (typeElision == AttrTypeElision::May && floatAttr.getType().isF64() &&
+        !printedHex)
       return;
 
   } else if (auto strAttr = llvm::dyn_cast<StringAttr>(attr)) {
diff --git a/mlir/lib/Query/QueryParser.cpp b/mlir/lib/Query/QueryParser.cpp
index 595055a42965ff475c59b2597c4beedbf54675f6..8a034634c5b89a29246038e6c7eec777fa5817cd 100644
--- a/mlir/lib/Query/QueryParser.cpp
+++ b/mlir/lib/Query/QueryParser.cpp
@@ -91,13 +91,11 @@ QueryRef QueryParser::endQuery(QueryRef queryRef) {
   llvm::StringRef extra = line;
   llvm::StringRef extraTrimmed = extra.ltrim(" \t\v\f\r");
 
-  if ((!extraTrimmed.empty() && extraTrimmed[0] == '\n') ||
-      (extraTrimmed.size() >= 2 && extraTrimmed[0] == '\r' &&
-       extraTrimmed[1] == '\n'))
+  if (extraTrimmed.starts_with('\n') || extraTrimmed.starts_with("\r\n"))
     queryRef->remainingContent = extra;
   else {
     llvm::StringRef trailingWord = lexWord();
-    if (!trailingWord.empty() && trailingWord.front() == '#') {
+    if (trailingWord.starts_with('#')) {
       line = line.drop_until([](char c) { return c == '\n'; });
       line = line.drop_while([](char c) { return c == '\n'; });
       return endQuery(queryRef);
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index a373c1fb5cdabd28e8e7c49c074c50da492de957..15737aa681c5937c26d29953d5b1783d34d1523c 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -113,6 +113,15 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) {
       node->getOffsetInBits(), node->getDWARFAddressSpace(), extraData);
 }
 
+DIStringTypeAttr DebugImporter::translateImpl(llvm::DIStringType *node) {
+  return DIStringTypeAttr::get(
+      context, node->getTag(), getStringAttrOrNull(node->getRawName()),
+      node->getSizeInBits(), node->getAlignInBits(),
+      translate(node->getStringLength()),
+      translateExpression(node->getStringLengthExp()),
+      translateExpression(node->getStringLocationExp()), node->getEncoding());
+}
+
 DIFileAttr DebugImporter::translateImpl(llvm::DIFile *node) {
   return DIFileAttr::get(context, node->getFilename(), node->getDirectory());
 }
@@ -174,6 +183,10 @@ DILocalVariableAttr DebugImporter::translateImpl(llvm::DILocalVariable *node) {
       node->getAlignInBits(), translate(node->getType()));
 }
 
+DIVariableAttr DebugImporter::translateImpl(llvm::DIVariable *node) {
+  return cast<DIVariableAttr>(translate(static_cast<llvm::DINode *>(node)));
+}
+
 DIScopeAttr DebugImporter::translateImpl(llvm::DIScope *node) {
   return cast<DIScopeAttr>(translate(static_cast<llvm::DINode *>(node)));
 }
@@ -295,6 +308,8 @@ DINodeAttr DebugImporter::translate(llvm::DINode *node) {
       return translateImpl(casted);
     if (auto *casted = dyn_cast<llvm::DIDerivedType>(node))
       return translateImpl(casted);
+    if (auto *casted = dyn_cast<llvm::DIStringType>(node))
+      return translateImpl(casted);
     if (auto *casted = dyn_cast<llvm::DIFile>(node))
       return translateImpl(casted);
     if (auto *casted = dyn_cast<llvm::DIGlobalVariable>(node))
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.h b/mlir/lib/Target/LLVMIR/DebugImporter.h
index 5f402fb0b657cc604a26f542c4fb9bc74950a6d9..fac86dbe2cdd22e325f968c4088e50f8b16fcd91 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.h
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.h
@@ -63,12 +63,14 @@ private:
   DICompileUnitAttr translateImpl(llvm::DICompileUnit *node);
   DICompositeTypeAttr translateImpl(llvm::DICompositeType *node);
   DIDerivedTypeAttr translateImpl(llvm::DIDerivedType *node);
+  DIStringTypeAttr translateImpl(llvm::DIStringType *node);
   DIFileAttr translateImpl(llvm::DIFile *node);
   DILabelAttr translateImpl(llvm::DILabel *node);
   DILexicalBlockAttr translateImpl(llvm::DILexicalBlock *node);
   DILexicalBlockFileAttr translateImpl(llvm::DILexicalBlockFile *node);
   DIGlobalVariableAttr translateImpl(llvm::DIGlobalVariable *node);
   DILocalVariableAttr translateImpl(llvm::DILocalVariable *node);
+  DIVariableAttr translateImpl(llvm::DIVariable *node);
   DIModuleAttr translateImpl(llvm::DIModule *node);
   DINamespaceAttr translateImpl(llvm::DINamespace *node);
   DIScopeAttr translateImpl(llvm::DIScope *node);
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index aef7a06f96c3475dc1c6bae335d398a9b0a5f394..35800e993d89d3f21a20f6106c754b42c733cbcc 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -182,6 +182,15 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) {
       /*Flags=*/llvm::DINode::FlagZero, translate(attr.getExtraData()));
 }
 
+llvm::DIStringType *DebugTranslation::translateImpl(DIStringTypeAttr attr) {
+  return llvm::DIStringType::get(
+      llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()),
+      translate(attr.getStringLength()),
+      getExpressionAttrOrNull(attr.getStringLengthExp()),
+      getExpressionAttrOrNull(attr.getStringLocationExp()),
+      attr.getSizeInBits(), attr.getAlignInBits(), attr.getEncoding());
+}
+
 llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) {
   return llvm::DIFile::get(llvmCtx, getMDStringOrNull(attr.getName()),
                            getMDStringOrNull(attr.getDirectory()));
@@ -210,6 +219,10 @@ llvm::DILocalScope *DebugTranslation::translateImpl(DILocalScopeAttr attr) {
   return cast<llvm::DILocalScope>(translate(DINodeAttr(attr)));
 }
 
+llvm::DIVariable *DebugTranslation::translateImpl(DIVariableAttr attr) {
+  return cast<llvm::DIVariable>(translate(DINodeAttr(attr)));
+}
+
 llvm::DILocalVariable *
 DebugTranslation::translateImpl(DILocalVariableAttr attr) {
   return llvm::DILocalVariable::get(
@@ -377,8 +390,8 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) {
                      DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr,
                      DILabelAttr, DILexicalBlockAttr, DILexicalBlockFileAttr,
                      DILocalVariableAttr, DIModuleAttr, DINamespaceAttr,
-                     DINullTypeAttr, DISubprogramAttr, DISubrangeAttr,
-                     DISubroutineTypeAttr>(
+                     DINullTypeAttr, DIStringTypeAttr, DISubprogramAttr,
+                     DISubrangeAttr, DISubroutineTypeAttr>(
                    [&](auto attr) { return translateImpl(attr); });
 
   if (node && !node->isTemporary())
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h
index 04b7ea41add9a0401bd0218d507f5c89b0c60505..16a853736226dad722b836885d40be1e27cc6257 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.h
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h
@@ -73,6 +73,7 @@ private:
   llvm::DICompileUnit *translateImpl(DICompileUnitAttr attr);
   llvm::DICompositeType *translateImpl(DICompositeTypeAttr attr);
   llvm::DIDerivedType *translateImpl(DIDerivedTypeAttr attr);
+  llvm::DIStringType *translateImpl(DIStringTypeAttr attr);
   llvm::DIFile *translateImpl(DIFileAttr attr);
   llvm::DILabel *translateImpl(DILabelAttr attr);
   llvm::DILexicalBlock *translateImpl(DILexicalBlockAttr attr);
@@ -80,6 +81,7 @@ private:
   llvm::DILocalScope *translateImpl(DILocalScopeAttr attr);
   llvm::DILocalVariable *translateImpl(DILocalVariableAttr attr);
   llvm::DIGlobalVariable *translateImpl(DIGlobalVariableAttr attr);
+  llvm::DIVariable *translateImpl(DIVariableAttr attr);
   llvm::DIModule *translateImpl(DIModuleAttr attr);
   llvm::DINamespace *translateImpl(DINamespaceAttr attr);
   llvm::DIScope *translateImpl(DIScopeAttr attr);
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index cfd4f9c03aaff24f8011778de02a0022fe9ba622..597cb29ce911bc3641746d2e76ba033f673cff86 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -319,8 +319,7 @@ private:
 /// This abstract class manages the worklist and contains helper methods for
 /// rewriting ops on the worklist. Derived classes specify how ops are added
 /// to the worklist in the beginning.
-class GreedyPatternRewriteDriver : public PatternRewriter,
-                                   public RewriterBase::Listener {
+class GreedyPatternRewriteDriver : public RewriterBase::Listener {
 protected:
   explicit GreedyPatternRewriteDriver(MLIRContext *ctx,
                                       const FrozenRewritePatternSet &patterns,
@@ -339,7 +338,8 @@ protected:
   /// Notify the driver that the specified operation was inserted. Update the
   /// worklist as needed: The operation is enqueued depending on scope and
   /// strict mode.
-  void notifyOperationInserted(Operation *op, InsertPoint previous) override;
+  void notifyOperationInserted(Operation *op,
+                               OpBuilder::InsertPoint previous) override;
 
   /// Notify the driver that the specified operation was removed. Update the
   /// worklist as needed: The operation and its children are removed from the
@@ -354,6 +354,10 @@ protected:
   /// reached. Return `true` if any IR was changed.
   bool processWorklist();
 
+  /// The pattern rewriter that is used for making IR modifications and is
+  /// passed to rewrite patterns.
+  PatternRewriter rewriter;
+
   /// The worklist for this transformation keeps track of the operations that
   /// need to be (re)visited.
 #ifdef MLIR_GREEDY_REWRITE_RANDOMIZER_SEED
@@ -407,7 +411,7 @@ private:
 GreedyPatternRewriteDriver::GreedyPatternRewriteDriver(
     MLIRContext *ctx, const FrozenRewritePatternSet &patterns,
     const GreedyRewriteConfig &config)
-    : PatternRewriter(ctx), config(config), matcher(patterns)
+    : rewriter(ctx), config(config), matcher(patterns)
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
       // clang-format off
       , expensiveChecks(
@@ -423,9 +427,9 @@ GreedyPatternRewriteDriver::GreedyPatternRewriteDriver(
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
   // Send IR notifications to the debug handler. This handler will then forward
   // all notifications to this GreedyPatternRewriteDriver.
-  setListener(&expensiveChecks);
+  rewriter.setListener(&expensiveChecks);
 #else
-  setListener(this);
+  rewriter.setListener(this);
 #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
 }
 
@@ -473,7 +477,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
 
     // If the operation is trivially dead - remove it.
     if (isOpTriviallyDead(op)) {
-      eraseOp(op);
+      rewriter.eraseOp(op);
       changed = true;
 
       LLVM_DEBUG(logResultWithLine("success", "operation is trivially dead"));
@@ -505,8 +509,8 @@ bool GreedyPatternRewriteDriver::processWorklist() {
         // Op results can be replaced with `foldResults`.
         assert(foldResults.size() == op->getNumResults() &&
                "folder produced incorrect number of results");
-        OpBuilder::InsertionGuard g(*this);
-        setInsertionPoint(op);
+        OpBuilder::InsertionGuard g(rewriter);
+        rewriter.setInsertionPoint(op);
         SmallVector<Value> replacements;
         bool materializationSucceeded = true;
         for (auto [ofr, resultType] :
@@ -519,7 +523,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
           }
           // Materialize Attributes as SSA values.
           Operation *constOp = op->getDialect()->materializeConstant(
-              *this, ofr.get<Attribute>(), resultType, op->getLoc());
+              rewriter, ofr.get<Attribute>(), resultType, op->getLoc());
 
           if (!constOp) {
             // If materialization fails, cleanup any operations generated for
@@ -532,7 +536,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
               replacementOps.insert(replacement.getDefiningOp());
             }
             for (Operation *op : replacementOps) {
-              eraseOp(op);
+              rewriter.eraseOp(op);
             }
 
             materializationSucceeded = false;
@@ -547,7 +551,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
         }
 
         if (materializationSucceeded) {
-          replaceOp(op, replacements);
+          rewriter.replaceOp(op, replacements);
           changed = true;
           LLVM_DEBUG(logSuccessfulFolding(dumpRootOp));
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
@@ -608,7 +612,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
 #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
 
     LogicalResult matchResult =
-        matcher.matchAndRewrite(op, *this, canApply, onFailure, onSuccess);
+        matcher.matchAndRewrite(op, rewriter, canApply, onFailure, onSuccess);
 
     if (succeeded(matchResult)) {
       LLVM_DEBUG(logResultWithLine("success", "pattern matched"));
@@ -664,8 +668,8 @@ void GreedyPatternRewriteDriver::notifyBlockErased(Block *block) {
     config.listener->notifyBlockErased(block);
 }
 
-void GreedyPatternRewriteDriver::notifyOperationInserted(Operation *op,
-                                                         InsertPoint previous) {
+void GreedyPatternRewriteDriver::notifyOperationInserted(
+    Operation *op, OpBuilder::InsertPoint previous) {
   LLVM_DEBUG({
     logger.startLine() << "** Insert  : '" << op->getName() << "'(" << op
                        << ")\n";
@@ -822,7 +826,7 @@ private:
 LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
   bool continueRewrites = false;
   int64_t iteration = 0;
-  MLIRContext *ctx = getContext();
+  MLIRContext *ctx = rewriter.getContext();
   do {
     // Check if the iteration limit was reached.
     if (++iteration > config.maxIterations &&
@@ -834,7 +838,7 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
 
     // `OperationFolder` CSE's constant ops (and may move them into parents
     // regions to enable more aggressive CSE'ing).
-    OperationFolder folder(getContext(), this);
+    OperationFolder folder(ctx, this);
     auto insertKnownConstant = [&](Operation *op) {
       // Check for existing constants when populating the worklist. This avoids
       // accidentally reversing the constant order during processing.
@@ -872,7 +876,7 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
           // After applying patterns, make sure that the CFG of each of the
           // regions is kept up to date.
           if (config.enableRegionSimplification)
-            continueRewrites |= succeeded(simplifyRegions(*this, region));
+            continueRewrites |= succeeded(simplifyRegions(rewriter, region));
         },
         {&region}, iteration);
   } while (continueRewrites);
diff --git a/mlir/python/mlir/ir.py b/mlir/python/mlir/ir.py
index 80c965b2d0eb2cc4f89abecb75ad16554d42dbd9..a9ac765fe1c178acf0853b22f3cfd0fe53b66d7d 100644
--- a/mlir/python/mlir/ir.py
+++ b/mlir/python/mlir/ir.py
@@ -68,7 +68,7 @@ def _si1Attr(x, context):
 
 
 @register_attribute_builder("SI8Attr")
-def _i8Attr(x, context):
+def _si8Attr(x, context):
     return IntegerAttr.get(IntegerType.get_signed(8, context=context), x)
 
 
@@ -93,7 +93,7 @@ def _ui1Attr(x, context):
 
 
 @register_attribute_builder("UI8Attr")
-def _i8Attr(x, context):
+def _ui8Attr(x, context):
     return IntegerAttr.get(IntegerType.get_unsigned(8, context=context), x)
 
 
diff --git a/mlir/test/CAPI/llvm.c b/mlir/test/CAPI/llvm.c
index f1d02b43d523208471ba97309f449a3681f83516..082ddc53e6282a960bb848fb7f05387118ddb294 100644
--- a/mlir/test/CAPI/llvm.c
+++ b/mlir/test/CAPI/llvm.c
@@ -293,8 +293,9 @@ static void testDebugInfoAttributes(MlirContext ctx) {
       mlirLLVMDILexicalBlockFileAttrGet(ctx, compile_unit, file, 3));
 
   // CHECK: #llvm.di_local_variable<{{.*}}>
-  mlirAttributeDump(mlirLLVMDILocalVariableAttrGet(ctx, compile_unit, foo, file,
-                                                   1, 0, 8, di_type));
+  MlirAttribute local_var = mlirLLVMDILocalVariableAttrGet(
+      ctx, compile_unit, foo, file, 1, 0, 8, di_type);
+  mlirAttributeDump(local_var);
   // CHECK: #llvm.di_derived_type<{{.*}}>
   // CHECK-NOT: dwarfAddressSpace
   mlirAttributeDump(mlirLLVMDIDerivedTypeAttrGet(
@@ -337,6 +338,12 @@ static void testDebugInfoAttributes(MlirContext ctx) {
   // CHECK: #llvm.di_expression<[(1)]>
   mlirAttributeDump(expression);
 
+  MlirAttribute string_type =
+      mlirLLVMDIStringTypeAttrGet(ctx, 0x0, foo, 16, 0, local_var, expression,
+                                  expression, MlirLLVMTypeEncodingSigned);
+  // CHECK: #llvm.di_string_type<{{.*}}>
+  mlirAttributeDump(string_type);
+
   // CHECK: #llvm.di_composite_type<{{.*}}>
   mlirAttributeDump(mlirLLVMDICompositeTypeAttrGet(
       ctx, 0, id, foo, file, 1, compile_unit, di_type, 0, 64, 8, 1, &di_type,
diff --git a/mlir/test/Conversion/FuncToEmitC/func-to-emitc.mlir b/mlir/test/Conversion/FuncToEmitC/func-to-emitc.mlir
index 5c96cf1ce0d34c7317cde30df375de7fcd288b1b..5730f7a4814fac89555ee3b38b80cb6adaa943cc 100644
--- a/mlir/test/Conversion/FuncToEmitC/func-to-emitc.mlir
+++ b/mlir/test/Conversion/FuncToEmitC/func-to-emitc.mlir
@@ -58,3 +58,19 @@ func.func @call(%arg0: i32) -> i32 {
 
 // CHECK-LABEL: emitc.func private @return_i32(i32) -> i32 attributes {specifiers = ["extern"]}
 func.func private @return_i32(%arg0: i32) -> i32
+
+// -----
+
+// CHECK-LABEL: emitc.func private @return_void() attributes {specifiers = ["static"]}
+// CHECK-NEXT: emitc.return
+func.func private @return_void() {
+  return
+}
+
+// CHECK-LABEL: emitc.func @call()
+// CHECK-NEXT: emitc.call @return_void() : () -> ()
+// CHECK-NEXT: emitc.return
+func.func @call() {
+  call @return_void() : () -> ()
+  return
+}
diff --git a/mlir/test/Conversion/SCFToEmitC/for.mlir b/mlir/test/Conversion/SCFToEmitC/for.mlir
index 7f90310af218942bdf0ee0128767227c9246afd8..7e59eac3d4095060371be0549838c54949478633 100644
--- a/mlir/test/Conversion/SCFToEmitC/for.mlir
+++ b/mlir/test/Conversion/SCFToEmitC/for.mlir
@@ -49,17 +49,13 @@ func.func @for_yield(%arg0 : index, %arg1 : index, %arg2 : index) -> (f32, f32)
 // CHECK-NEXT:    %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK-NEXT:    %[[VAL_5:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
 // CHECK-NEXT:    %[[VAL_6:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:    %[[VAL_7:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:    %[[VAL_8:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:    emitc.assign %[[VAL_3]] : f32 to %[[VAL_7]] : f32
-// CHECK-NEXT:    emitc.assign %[[VAL_4]] : f32 to %[[VAL_8]] : f32
+// CHECK-NEXT:    emitc.assign %[[VAL_3]] : f32 to %[[VAL_5]] : f32
+// CHECK-NEXT:    emitc.assign %[[VAL_4]] : f32 to %[[VAL_6]] : f32
 // CHECK-NEXT:    emitc.for %[[VAL_9:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_2]] {
-// CHECK-NEXT:      %[[VAL_10:.*]] = arith.addf %[[VAL_7]], %[[VAL_8]] : f32
-// CHECK-NEXT:      emitc.assign %[[VAL_10]] : f32 to %[[VAL_7]] : f32
-// CHECK-NEXT:      emitc.assign %[[VAL_10]] : f32 to %[[VAL_8]] : f32
+// CHECK-NEXT:      %[[VAL_10:.*]] = arith.addf %[[VAL_5]], %[[VAL_6]] : f32
+// CHECK-NEXT:      emitc.assign %[[VAL_10]] : f32 to %[[VAL_5]] : f32
+// CHECK-NEXT:      emitc.assign %[[VAL_10]] : f32 to %[[VAL_6]] : f32
 // CHECK-NEXT:    }
-// CHECK-NEXT:    emitc.assign %[[VAL_7]] : f32 to %[[VAL_5]] : f32
-// CHECK-NEXT:    emitc.assign %[[VAL_8]] : f32 to %[[VAL_6]] : f32
 // CHECK-NEXT:    return %[[VAL_5]], %[[VAL_6]] : f32, f32
 // CHECK-NEXT:  }
 
@@ -78,19 +74,15 @@ func.func @nested_for_yield(%arg0 : index, %arg1 : index, %arg2 : index) -> f32
 // CHECK-SAME:      %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index) -> f32 {
 // CHECK-NEXT:    %[[VAL_3:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK-NEXT:    %[[VAL_4:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:    %[[VAL_5:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:    emitc.assign %[[VAL_3]] : f32 to %[[VAL_5]] : f32
+// CHECK-NEXT:    emitc.assign %[[VAL_3]] : f32 to %[[VAL_4]] : f32
 // CHECK-NEXT:    emitc.for %[[VAL_6:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_2]] {
 // CHECK-NEXT:      %[[VAL_7:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:      %[[VAL_8:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
-// CHECK-NEXT:      emitc.assign %[[VAL_5]] : f32 to %[[VAL_8]] : f32
+// CHECK-NEXT:      emitc.assign %[[VAL_4]] : f32 to %[[VAL_7]] : f32
 // CHECK-NEXT:      emitc.for %[[VAL_9:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_2]] {
-// CHECK-NEXT:        %[[VAL_10:.*]] = arith.addf %[[VAL_8]], %[[VAL_8]] : f32
-// CHECK-NEXT:        emitc.assign %[[VAL_10]] : f32 to %[[VAL_8]] : f32
+// CHECK-NEXT:        %[[VAL_10:.*]] = arith.addf %[[VAL_7]], %[[VAL_7]] : f32
+// CHECK-NEXT:        emitc.assign %[[VAL_10]] : f32 to %[[VAL_7]] : f32
 // CHECK-NEXT:      }
-// CHECK-NEXT:      emitc.assign %[[VAL_8]] : f32 to %[[VAL_7]] : f32
-// CHECK-NEXT:      emitc.assign %[[VAL_7]] : f32 to %[[VAL_5]] : f32
+// CHECK-NEXT:      emitc.assign %[[VAL_7]] : f32 to %[[VAL_4]] : f32
 // CHECK-NEXT:    }
-// CHECK-NEXT:    emitc.assign %[[VAL_5]] : f32 to %[[VAL_4]] : f32
 // CHECK-NEXT:    return %[[VAL_4]] : f32
 // CHECK-NEXT:  }
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 12121ea0dd70e8864bce264a6f7ead19e4ac44f8..54dcf07053906f6e011eba041ac796a02c8af901 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -2565,6 +2565,22 @@ func.func @vector_deinterleave_1d_scalable(%a: vector<[4]xi32>) -> (vector<[2]xi
     return %0, %1 : vector<[2]xi32>, vector<[2]xi32>
 }
 
+// CHECK-LABEL: @vector_deinterleave_2d
+// CHECK-SAME: %[[SRC:.*]]: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>)
+func.func @vector_deinterleave_2d(%a: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) {
+  // CHECK: llvm.shufflevector
+  // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x8xf32>
+  %0, %1 = vector.deinterleave %a : vector<2x8xf32> -> vector<2x4xf32>
+  return %0, %1 : vector<2x4xf32>, vector<2x4xf32>
+}
+
+func.func @vector_deinterleave_2d_scalable(%a: vector<2x[8]xf32>) -> (vector<2x[4]xf32>, vector<2x[4]xf32>) {
+    // CHECK: llvm.intr.vector.deinterleave2
+    // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x[8]xf32>
+    %0, %1 = vector.deinterleave %a : vector<2x[8]xf32> -> vector<2x[4]xf32>
+    return %0, %1 : vector<2x[4]xf32>, vector<2x[4]xf32>
+}
+
 // -----
 
 // CHECK-LABEL: func.func @vector_bitcast_2d
diff --git a/mlir/test/Dialect/ArmSME/enable-arm-streaming-invalid.mlir b/mlir/test/Dialect/ArmSME/enable-arm-streaming-invalid.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..da70b632d70c4e8a45f3f01e2a3dae4d52258d80
--- /dev/null
+++ b/mlir/test/Dialect/ArmSME/enable-arm-streaming-invalid.mlir
@@ -0,0 +1,4 @@
+// RUN: mlir-opt %s -enable-arm-streaming="if-contains-scalable-vectors if-required-by-ops" -verify-diagnostics
+
+// expected-error@below {{enable-arm-streaming: `if-required-by-ops` and `if-contains-scalable-vectors` are mutually exclusive}}
+func.func @test() { return }
diff --git a/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir b/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
index 6b58d8fdc41b0e070e8e52098cd5e951edc42930..2011802c5c8b2f08f112ee7240915501b86281dc 100644
--- a/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
+++ b/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
@@ -2,7 +2,8 @@
 // RUN: mlir-opt %s -enable-arm-streaming=streaming-mode=streaming-locally -verify-diagnostics | FileCheck %s -check-prefix=CHECK-LOCALLY
 // RUN: mlir-opt %s -enable-arm-streaming=streaming-mode=streaming-compatible -verify-diagnostics | FileCheck %s -check-prefix=CHECK-COMPATIBLE
 // RUN: mlir-opt %s -enable-arm-streaming=za-mode=new-za -verify-diagnostics | FileCheck %s -check-prefix=CHECK-ENABLE-ZA
-// RUN: mlir-opt %s -enable-arm-streaming=only-if-required-by-ops -verify-diagnostics | FileCheck %s -check-prefix=IF-REQUIRED
+// RUN: mlir-opt %s -enable-arm-streaming=if-required-by-ops -verify-diagnostics | FileCheck %s -check-prefix=IF-REQUIRED
+// RUN: mlir-opt %s -enable-arm-streaming=if-contains-scalable-vectors -verify-diagnostics | FileCheck %s -check-prefix=IF-SCALABLE
 
 // CHECK-LABEL: @arm_streaming
 // CHECK-SAME: attributes {arm_streaming}
@@ -38,3 +39,17 @@ func.func @requires_arm_streaming() {
 // IF-REQUIRED: @does_not_require_arm_streaming
 // IF-REQUIRED-NOT: arm_streaming
 func.func @does_not_require_arm_streaming() { return }
+
+// IF-SCALABLE-LABEL: @contains_scalable_vectors
+// IF-SCALABLE-SAME: attributes {arm_streaming}
+func.func @contains_scalable_vectors(%vec: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.addf %vec, %vec : vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+
+// IF-SCALABLE-LABEL: @no_scalable_vectors
+// IF-SCALABLE-NOT: arm_streaming
+func.func @no_scalable_vectors(%vec: vector<4xf32>) -> vector<4xf32> {
+  %0 = arith.addf %vec, %vec : vector<4xf32>
+  return %0 : vector<4xf32>
+}
diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-callop-interface.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-callop-interface.mlir
index 63947150c23b30e8a2b14aa2ab3f0de664477c63..a77442dfe40e5b9bdfc4532f5282887d9aadf3aa 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-callop-interface.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-callop-interface.mlir
@@ -131,3 +131,22 @@ func.func @g(%arg0: memref<f32>) -> memref<f32> {
 // CHECK-DYNAMIC-LABEL: func private @f
 //  CHECK-DYNAMIC-SAME: (memref<f32>) -> memref<f32>
 //       CHECK-DYNAMIC: call @f({{.*}}) : (memref<f32>) -> memref<f32>
+
+// -----
+
+func.func @func_call_indirect(%m: memref<?xf32>, %f: (memref<?xf32>) -> (memref<?xf32>)) {
+  %0 = func.call_indirect %f(%m) : (memref<?xf32>) -> (memref<?xf32>)
+  return
+}
+
+// CHECK-LABEL: func @func_call_indirect(
+//       CHECK:   %[[true:.*]] = arith.constant true
+//       CHECK:   %[[call:.*]] = call_indirect {{.*}} : (memref<?xf32>) -> memref<?xf32>
+//       CHECK:   %[[base_call:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[call]]
+//       CHECK:   bufferization.dealloc (%[[base_call]] : {{.*}}) if (%[[true]])
+
+// CHECK-DYNAMIC-LABEL: func @func_call_indirect(
+//       CHECK-DYNAMIC:   %[[true:.*]] = arith.constant true
+//       CHECK-DYNAMIC:   %[[call:.*]] = call_indirect {{.*}} : (memref<?xf32>) -> memref<?xf32>
+//       CHECK-DYNAMIC:   %[[base_call:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[call]]
+//       CHECK-DYNAMIC:   bufferization.dealloc (%[[base_call]] : {{.*}}) if (%[[true]])
diff --git a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
index 0d40df534a3bb7f2e7b6048a80a0d41c6aae1cf6..600f0dea31f4a860c390ae70b1b7da29ecb2bab3 100644
--- a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
+++ b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir
@@ -537,3 +537,71 @@ func.func @no_fold_non_consecutive_reduction_dims(%arg0 : tensor<?x?xi32>, %sz0:
 //      CHECK:   %[[GENERIC:.+]] = linalg.generic
 // CHECK-SAME:       ins(%[[EXPAND_ARG0]] :
 //      CHECK:   return %[[GENERIC]]
+
+// -----
+
+func.func @fuse_by_collapsing_pad(%arg0 : tensor<2x12x5x336x9xi32>) -> tensor<8x3x4x17x6x7x8x14xi32> {
+  %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32>
+  %cst = arith.constant 0 : i32
+  %padded_0 = tensor.pad %expand low[1, 0, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index,
+       %arg5: index, %arg6: index, %arg7: index, %arg8: index):
+    tensor.yield %cst : i32
+  } : tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x3x4x17x6x7x8x14xi32>
+  return %padded_0 : tensor<8x3x4x17x6x7x8x14xi32>
+}
+//      CHECK: func @fuse_by_collapsing_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>)
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[ARG0]]
+// CHECK-SAME:       low[1, 0, 8, 0, 3] high[5, 0, 4, 0, 2]
+//      CHECK:       tensor<2x12x5x336x9xi32> to tensor<8x12x17x336x14xi32>
+//      CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]]
+// CHECK-SAME:       output_shape [8, 3, 4, 17, 6, 7, 8, 14] : tensor<8x12x17x336x14xi32> into tensor<8x3x4x17x6x7x8x14xi32>
+//      CHECK:   return %[[EXPAND]]
+
+// -----
+
+func.func @no_fuse_by_collapsing_pad(%arg0 : tensor<2x12x5x336x9xi32>) -> tensor<8x5x4x17x6x7x8x14xi32> {
+  %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32>
+  %cst = arith.constant 0 : i32
+  %padded_0 = tensor.pad %expand low[1, 2, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index,
+       %arg5: index, %arg6: index, %arg7: index, %arg8: index):
+    tensor.yield %cst : i32
+  } : tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x5x4x17x6x7x8x14xi32>
+  return %padded_0 : tensor<8x5x4x17x6x7x8x14xi32>
+}
+//      CHECK: func @no_fuse_by_collapsing_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>)
+//      CHECK:   %[[EXPAND_ARG0:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]]
+// CHECK-SAME:       output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32>
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[EXPAND_ARG0]]
+// CHECK-SAME:       low[1, 2, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2]
+//      CHECK:       tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x5x4x17x6x7x8x14xi32>
+//      CHECK:   return %[[PAD]]
+
+// -----
+
+func.func @fuse_by_collapsing_dynamic_pad(%arg0 : tensor<?x?x?x?xf32>,
+    %s0 : index, %s1 : index, %s2 : index, %s3 : index, %s4 : index, %s5 : index,
+    %l0 : index, %l1 : index, %h0 : index, %h1 : index) -> tensor<?x?x?x?x?x?xf32> {
+  %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5]] output_shape [%s0, %s1, %s2, %s3, %s4, %s5] : tensor<?x?x?x?xf32> into tensor<?x?x?x?x?x?xf32>
+  %cst = arith.constant 0.0 : f32
+  %padded_0 = tensor.pad %expand low[%l0, 0, 0, %l1, 0, 0] high[%h0, 0, 0, %h1, 0, 0] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index):
+    tensor.yield %cst : f32
+  } : tensor<?x?x?x?x?x?xf32> to tensor<?x?x?x?x?x?xf32>
+  return %padded_0 : tensor<?x?x?x?x?x?xf32>
+}
+//  CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1, s2] -> (s0 + s1 + s2)>
+//      CHECK: func @fuse_by_collapsing_dynamic_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<?x?x?x?xf32>
+// CHECK-SAME:   %[[S0:.+]]: index, %[[S1:.+]]: index, %[[S2:.+]]: index, %[[S3:.+]]: index, %[[S4:.+]]: index, %[[S5:.+]]: index, %[[L0:.+]]: index, %[[L1:.+]]: index, %[[H0:.+]]: index, %[[H1:.+]]: index
+//      CHECK:   %[[PAD_SIZE0:.+]] = affine.apply #[[MAP]]()[%[[L0]], %[[H0]], %[[S0]]]
+//      CHECK:   %[[PAD_SIZE1:.+]] = affine.apply #[[MAP]]()[%[[L1]], %[[H1]], %[[S3]]]
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[ARG0]]
+// CHECK-SAME:       low[%[[L0]], 0, %[[L1]], 0] high[%[[H0]], 0, %[[H1]], 0]
+//      CHECK:       tensor<?x?x?x?xf32> to tensor<?x?x?x?xf32>
+//      CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5]]
+// CHECK-SAME:       output_shape [%[[PAD_SIZE0]], %[[S1]], %[[S2]], %[[PAD_SIZE1]], %[[S4]], %[[S5]]] : tensor<?x?x?x?xf32> into tensor<?x?x?x?x?x?xf32>
+//      CHECK:   return %[[EXPAND]]
diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
index f42666f81bbadd1dcb2e0a88ebbebd0da22d2712..b8df5fc88e1999ea63fc9118b5d7d29d6da4931b 100644
--- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
@@ -826,3 +826,64 @@ func.func @linalg_add_reshape_producer_fusion(%arg0 : tensor<?x7x?x8xf32>,
 // CHECK-SAME:     [0, 1], [2, 3]
 // CHECK-SAME:     tensor<?x7x?x8xf32> into tensor<?x?xf32>
 //      CHECK:   return %[[T4]]
+
+// -----
+
+func.func @fuse_by_expanding_pad(%arg0 : tensor<2x3x4x5x6x7x8x9xi32>) -> tensor<8x12x17x336x14xi32> {
+  %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32>
+  %cst = arith.constant 0 : i32
+  %padded_0 = tensor.pad %collapse low[1, 0, 8, 0, 3] high[5, 0, 4, 0, 2] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index):
+    tensor.yield %cst : i32
+  } : tensor<2x12x5x336x9xi32> to tensor<8x12x17x336x14xi32>
+  return %padded_0 : tensor<8x12x17x336x14xi32>
+}
+//      CHECK: func @fuse_by_expanding_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x3x4x5x6x7x8x9xi32>)
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[ARG0]]
+// CHECK-SAME:       low[1, 0, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2]
+//      CHECK:       tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x3x4x17x6x7x8x14xi32>
+//      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]]
+// CHECK-SAME:       : tensor<8x3x4x17x6x7x8x14xi32> into tensor<8x12x17x336x14xi32>
+//      CHECK:   return %[[COLLAPSE]]
+
+// -----
+
+func.func @no_fuse_by_expanding_pad(%arg0 : tensor<2x3x4x5x6x7x8x9xi32>) -> tensor<8x12x17x339x14xi32> {
+  %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32>
+  %cst = arith.constant 0 : i32
+  %padded_0 = tensor.pad %collapse low[1, 0, 8, 0, 3] high[5, 0, 4, 3, 2] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index):
+    tensor.yield %cst : i32
+  } : tensor<2x12x5x336x9xi32> to tensor<8x12x17x339x14xi32>
+  return %padded_0 : tensor<8x12x17x339x14xi32>
+}
+//      CHECK: func @no_fuse_by_expanding_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x3x4x5x6x7x8x9xi32>)
+//      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]]
+// CHECK-SAME:       : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32>
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[COLLAPSE]]
+// CHECK-SAME:       low[1, 0, 8, 0, 3] high[5, 0, 4, 3, 2]
+//      CHECK:       tensor<2x12x5x336x9xi32> to tensor<8x12x17x339x14xi32>
+//      CHECK:   return %[[PAD]]
+
+// -----
+
+func.func @fuse_by_expanding_dynamic_pad(%arg0 : tensor<?x?x?x?x?x?xi32>, %l0: index, %l1: index, %h0: index, %h1: index) -> tensor<?x?x?x?xi32> {
+  %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5]] : tensor<?x?x?x?x?x?xi32> into tensor<?x?x?x?xi32>
+  %cst = arith.constant 0 : i32
+  %padded_0 = tensor.pad %collapse low[%l0, 0, %l1, 0] high[%h0, 0, %h1, 0] {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index):
+    tensor.yield %cst : i32
+  } : tensor<?x?x?x?xi32> to tensor<?x?x?x?xi32>
+  return %padded_0 : tensor<?x?x?x?xi32>
+}
+//      CHECK: func @fuse_by_expanding_dynamic_pad(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<?x?x?x?x?x?xi32>
+// CHECK-SAME:   %[[L0:.+]]: index, %[[L1:.+]]: index, %[[H0:.+]]: index, %[[H1:.+]]: index
+//      CHECK:   %[[PAD:.+]] = tensor.pad %[[ARG0]]
+// CHECK-SAME:       low[%[[L0]], 0, 0, %[[L1]], 0, 0] high[%[[H0]], 0, 0, %[[H1]], 0, 0]
+//      CHECK:       tensor<?x?x?x?x?x?xi32> to tensor<?x?x?x?x?x?xi32>
+//      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5]]
+// CHECK-SAME:       : tensor<?x?x?x?x?x?xi32> into tensor<?x?x?x?xi32>
+//      CHECK:   return %[[COLLAPSE]]
diff --git a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
index 926969bfc73880e0c21f4bd5a54e0053b6c3d929..f34ef4f961483d70be0c535d83c683a6d5cf1b1a 100644
--- a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
+++ b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
@@ -622,9 +622,20 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// At the moment, we cannot lower tensor.unpack with outer_dims_perm.
-func.func @diagnostic_unpack(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> {
-  // expected-note @below {{target payload op}}
+// CHECK-LABEL: @unpack_with_outer_dims_perm
+//  CHECK-SAME: %[[ARG0:.*]]: tensor<32x64xf32>, %[[ARG1:.*]]: tensor<2x4x32x8xf32>
+//       CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<4x8x2x32xf32>
+//       CHECK: %[[TRAN:.*]] = linalg.transpose 
+//  CHECK-SAME:   ins(%[[ARG1]] : tensor<2x4x32x8xf32>) 
+//  CHECK-SAME:   outs(%[[EMPTY]] : tensor<4x8x2x32xf32>) 
+//  CHECK-SAME:   permutation = [1, 3, 0, 2]
+//       CHECK: %[[CLP:.*]] = tensor.collapse_shape %[[TRAN]] {{\[}}[0, 1], [2, 3]]
+//  CHECK-SAME:   : tensor<4x8x2x32xf32> into tensor<32x64xf32>
+//       CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[CLP]][0, 0] [32, 64] [1, 1]
+//  CHECK-SAME:   : tensor<32x64xf32> to tensor<32x64xf32>
+//       CHECK: linalg.copy ins(%[[SLICE]]
+//  CHECK-SAME:   : tensor<32x64xf32>) outs(%[[ARG0]] : tensor<32x64xf32>) -> tensor<32x64xf32>
+func.func @unpack_with_outer_dims_perm(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> {
   %unpack = tensor.unpack %arg1 outer_dims_perm = [1, 0] 
     inner_dims_pos = [1, 0] inner_tiles = [32, 8] into %arg0 : tensor<2x4x32x8xf32> -> tensor<32x64xf32>
   return %unpack : tensor<32x64xf32>
@@ -634,7 +645,6 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op
       : (!transform.any_op) -> !transform.op<"tensor.unpack">
-    // expected-error @below {{cannot lower to transpose + collapse + extract}} 
     transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">)
       -> (!transform.op<"tensor.empty">,
           !transform.op<"linalg.transpose">,
diff --git a/mlir/test/Dialect/SCF/transform-ops.mlir b/mlir/test/Dialect/SCF/transform-ops.mlir
index a4daa86583c3da300ad9c033913e17543dbcaa67..2d6b48fd3e57c2d8b457e619947cc228780f0e8d 100644
--- a/mlir/test/Dialect/SCF/transform-ops.mlir
+++ b/mlir/test/Dialect/SCF/transform-ops.mlir
@@ -300,3 +300,60 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+
+// -----
+ 
+// CHECK-LABEL: func.func @loop_pipeline
+func.func @loop_pipeline(%arg0: memref<4x16xf32>, %arg1: vector<16xf32>) -> vector<16xf32> {
+   %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %cst = arith.constant 0.000000e+00 : f32
+  %c3 = arith.constant 3 : index
+  // CHECK: vector.transfer_read
+  // CHECK: vector.transfer_read
+  // CHECK: vector.transfer_read
+  // CHECK: arith.addf
+  // CHECK: arith.addf
+  // CHECK: arith.addf
+  %0 = scf.for %arg2 = %c0 to %c3 step %c1 iter_args(%arg3 = %arg1) -> (vector<16xf32>) {
+    %1 = vector.transfer_read %arg0[%arg2, %c0], %cst {in_bounds = [true]} : memref<4x16xf32>, vector<16xf32>
+    %2 = arith.addf %1, %arg3 : vector<16xf32>
+    scf.yield %2 : vector<16xf32>
+  }
+  return %0 : vector<16xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.op<"scf.for">
+    %1 = transform.loop.pipeline %0 {iteration_interval = 1 : i64, read_latency = 5 : i64,  scheduling_type = "full-loops"} : (!transform.op<"scf.for">) -> !transform.any_op
+     transform.yield
+ }
+}
+ 
+ 
+// -----
+ 
+// CHECK-LABEL: func.func @loop_pipeline_lb_gt_0
+func.func @loop_pipeline_lb_gt_0(%arg0: memref<4x16xf32>, %arg1: vector<16xf32>) -> vector<16xf32> {
+  %c1 = arith.constant 1 : index
+  %cst = arith.constant 0.000000e+00 : f32
+  %c3 = arith.constant 3 : index
+  // CHECK: vector.transfer_read
+  // CHECK: vector.transfer_read
+  // CHECK: arith.addf
+  // CHECK: arith.addf
+  %0 = scf.for %arg2 = %c1 to %c3 step %c1 iter_args(%arg3 = %arg1) -> (vector<16xf32>) {
+    %1 = vector.transfer_read %arg0[%arg2, %c1], %cst {in_bounds = [true]} : memref<4x16xf32>, vector<16xf32>
+    %2 = arith.addf %1, %arg3 : vector<16xf32>
+    scf.yield %2 : vector<16xf32>
+  }
+  return %0 : vector<16xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.op<"scf.for">
+    %1 = transform.loop.pipeline %0 {iteration_interval = 1 : i64, read_latency = 5 : i64,  scheduling_type = "full-loops"} : (!transform.op<"scf.for">) -> !transform.any_op
+     transform.yield
+ }
+}
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index f7fbd3834288bf17044ac220d7540805a07768bd..df2bea08577e221df63d15d213d78fa4e6c6aa95 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -1139,7 +1139,7 @@ func.func @fold_collapse_of_expand(%arg0 : tensor<12x4xf32>) -> tensor<12x4xf32>
   return %1 : tensor<12x4xf32>
 }
 // CHECK-LABEL: @fold_collapse_of_expand
-//   CHECK-NOT:   linalg.{{.*}}shape
+//   CHECK-NOT:   tensor.{{.*}}_shape
 
 // -----
 
@@ -1152,7 +1152,75 @@ func.func @fold_collapse_of_expand_dynamic(%arg0 : tensor<?x?xf32>, %arg1: index
   return %1 : tensor<?x?xf32>
 }
 // CHECK-LABEL: @fold_collapse_of_expand_dynamic
-//   CHECK-NOT:   linalg.{{.*}}_shape
+//   CHECK-NOT:   tensor.{{.*}}_shape
+
+// -----
+
+func.func @fold_collapse_of_expand_fully_dynamic(%arg0 : tensor<?x?xf32>, %arg1: index, %arg2: index, %arg3: index)
+    -> tensor<?x?xf32> {
+  %0 = tensor.expand_shape %arg0 [[0, 1], [2]] output_shape [%arg1, %arg2, %arg3]
+      : tensor<?x?xf32> into tensor<?x?x?xf32>
+  %1 = tensor.collapse_shape %0 [[0, 1], [2]]
+      : tensor<?x?x?xf32> into tensor<?x?xf32>
+  return %1 : tensor<?x?xf32>
+}
+// CHECK-LABEL: @fold_collapse_of_expand_fully_dynamic
+//   CHECK-NOT:   tensor.{{.*}}_shape
+
+// -----
+
+func.func @no_fold_parallel_collapse_of_expand_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1: index, %arg2: index, %arg3: index, %arg4: index)
+    -> tensor<?x?x?xf32> {
+  %0 = tensor.expand_shape %arg0 [[0, 1], [2], [3]] output_shape [%arg1, %arg2, %arg3, %arg4]
+      : tensor<?x?x?xf32> into tensor<?x?x?x?xf32>
+  %1 = tensor.collapse_shape %0 [[0], [1], [2, 3]]
+      : tensor<?x?x?x?xf32> into tensor<?x?x?xf32>
+  return %1 : tensor<?x?x?xf32>
+}
+// CHECK-LABEL: @no_fold_parallel_collapse_of_expand_dynamic
+//       CHECK:   tensor.expand_shape
+//       CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape
+//       CHECK:   return %[[COLLAPSE]]
+
+// -----
+
+func.func @fold_expand_of_collapse(%arg0 : tensor<3x4x4xf32>) -> tensor<3x4x4xf32> {
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2]]
+      : tensor<3x4x4xf32> into tensor<12x4xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [3, 4, 4]
+      : tensor<12x4xf32> into tensor<3x4x4xf32>
+  return %1 : tensor<3x4x4xf32>
+}
+// CHECK-LABEL: @fold_expand_of_collapse
+//   CHECK-NOT:   tensor.{{.*}}_shape
+
+// -----
+
+func.func @fold_expand_of_collapse_dynamic(%arg0 : tensor<?x4x?xf32>, %arg1: index, %arg2: index)
+    -> tensor<?x4x?xf32> {
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2]]
+      : tensor<?x4x?xf32> into tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, 4, %arg2]
+      : tensor<?x?xf32> into tensor<?x4x?xf32>
+  return %1 : tensor<?x4x?xf32>
+}
+// CHECK-LABEL: @fold_expand_of_collapse_dynamic
+//   CHECK-NOT:   tensor.{{.*}}_shape
+
+// -----
+
+func.func @no_fold_expand_of_collapse_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1: index, %arg2: index, %arg3: index)
+    -> tensor<?x?x?xf32> {
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2]]
+      : tensor<?x?x?xf32> into tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, %arg2, %arg3]
+      : tensor<?x?xf32> into tensor<?x?x?xf32>
+  return %1 : tensor<?x?x?xf32>
+}
+// CHECK-LABEL: @no_fold_expand_of_collapse_dynamic
+//       CHECK:   tensor.collapse_shape
+//       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape
+//       CHECK:   return %[[EXPAND]]
 
 // -----
 
@@ -2523,3 +2591,18 @@ func.func @dim_out_of_bounds() -> vector<7xi32> {
     %16 = affine.vector_load %alloc_21[%c1, %c1, %dim] : memref<?x26x2xi32>, vector<7xi32>
     return %16 : vector<7xi32>
 }
+
+// -----
+
+// CHECK-LABEL:   func.func @test_destination_multiple_result(
+// CHECK-SAME:         %[[ARG1:.*]]: tensor<2x2xf32>,
+// CHECK-SAME:         %[[ARG2:.*]]: tensor<2x2xf32>) -> index {
+// CHECK:           %[[RES:.*]]:2 = test.destination_style_op ins(%[[ARG1]] : tensor<2x2xf32>)
+// CHECK-SAME:      outs(%[[ARG2]] : tensor<2x2xf32>) -> tensor<2x2xf32>, index
+// CHECK:           return %[[RES]]#1 : index
+func.func @test_destination_multiple_result(%arg0: tensor<2x2xf32>, %arg1: tensor<2x2xf32>) -> index {
+  %cast = tensor.cast %arg0 : tensor<2x2xf32> to tensor<?x2xf32>
+  %cast_0 = tensor.cast %arg1 : tensor<2x2xf32> to tensor<?x2xf32>
+  %0:2 = test.destination_style_op ins(%cast : tensor<?x2xf32>) outs(%cast_0 : tensor<?x2xf32>) -> tensor<?x2xf32>, index
+  return %0#1 : index
+}
diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
index 9a3143f5e550e1235b59065af45efa57c3065eaa..629a4c213572073c05d8ae2630e5b70b8366d47e 100644
--- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
+++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
@@ -636,3 +636,142 @@ func.func @tensor_padded_unpack_linalg_transpose_fold(%arg0: tensor<71x7x4x16x16
 //  CHECK-SAME:        into %[[OUT:.+]] : tensor<71x7x4x16x16xf32> -> tensor<100x71x64xf32>
 //       CHECK:       return %[[UNPACK]] : tensor<100x71x64xf32>
 //       CHECK:    }
+
+// -----
+
+func.func @non_involution_transpose_unpack_fold(%arg0: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> {
+  %0 = tensor.empty() : tensor<5x2x3x16x4xi32>
+  %transposed = linalg.transpose ins(%arg0 : tensor<2x3x5x4x16xi32>)
+                outs(%0 : tensor<5x2x3x16x4xi32>)
+                permutation = [2, 0, 1, 4, 3]
+  %1 = tensor.empty() : tensor<5x48x8xi32>
+  %unpack = tensor.unpack %transposed
+            outer_dims_perm = [0, 2, 1]
+            inner_dims_pos = [1, 2]
+            inner_tiles = [16, 4] into
+            %1 : tensor<5x2x3x16x4xi32> -> tensor<5x48x8xi32>
+  return %unpack : tensor<5x48x8xi32>
+}
+//CHECK-LABEL:  func.func @non_involution_transpose_unpack_fold(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> {
+//      CHECK:     %[[OUT:.+]] = tensor.empty() : tensor<5x48x8xi32>
+//      CHECK:     %[[UNPACK:.+]] = tensor.unpack %[[ARG0]]
+// CHECK-SAME:        outer_dims_perm = [2, 1, 0]
+// CHECK-SAME:        inner_dims_pos = [2, 1]
+// CHECK-SAME:        inner_tiles = [4, 16]
+// CHEKC-SAME:        into %[[OUT]] : tensor<2x3x5x4x16xi32> -> tensor<5x48x8xi32>
+//      CHECK:     return %[[UNPACK]] : tensor<5x48x8xi32>
+//      CHECK:   }
+
+// -----
+
+func.func @unpack_non_involution_transpose_fold(%arg0: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> {
+  %0 = tensor.empty() : tensor<3x56x3648xf32>
+  %unpack = tensor.unpack %arg0
+    outer_dims_perm = [2, 0, 1]
+    inner_dims_pos = [1, 2]
+    inner_tiles = [1, 64]
+    into %0 : tensor<57x3x56x1x64xf32> -> tensor<3x56x3648xf32>
+
+  %1 = tensor.empty() : tensor<3648x3x56xf32>
+  %transposed = linalg.transpose
+    ins(%unpack : tensor<3x56x3648xf32>)
+    outs(%1 : tensor<3648x3x56xf32>)
+    permutation = [2, 0, 1]
+  return %transposed : tensor<3648x3x56xf32>
+}
+// CHECK-LABEL:  func.func @unpack_non_involution_transpose_fold(
+//  CHECK-SAME:    %[[ARG0:.+]]: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> {
+//       CHECK:        %[[OUT:.+]] = tensor.empty() : tensor<3648x3x56xf32>
+//       CHECK:        %[[UNPACK:.+]] = tensor.unpack %[[ARG0]]
+//  CHECK-SAME:        outer_dims_perm = [0, 1, 2]
+//  CHECK-SAME:        inner_dims_pos = [2, 0]
+//  CHECK-SAME:        inner_tiles = [1, 64]
+//  CHECK-SAME:        into %[[OUT:.+]] : tensor<57x3x56x1x64xf32> -> tensor<3648x3x56xf32>
+//       CHECK:       return %[[UNPACK]] : tensor<3648x3x56xf32>
+//       CHECK:    }
+
+// -----
+
+func.func @transpose_unpacked_dims_no_fold(%arg0: tensor<2x16x5x4x3xi32>) -> tensor<5x32x12xi32> {
+  %0 = tensor.empty() : tensor<5x2x3x16x4xi32>
+  %transposed = linalg.transpose ins(%arg0 : tensor<2x16x5x4x3xi32>)
+                outs(%0 : tensor<5x2x3x16x4xi32>)
+                permutation = [2, 0, 4, 1, 3]
+  %1 = tensor.empty() : tensor<5x32x12xi32>
+  %unpack = tensor.unpack %transposed
+            inner_dims_pos = [1, 2]
+            inner_tiles = [16, 4] into
+            %1 : tensor<5x2x3x16x4xi32> -> tensor<5x32x12xi32>
+  return %unpack : tensor<5x32x12xi32>
+}
+//CHECK-LABEL:  func.func @transpose_unpacked_dims_no_fold(
+//      CHECK:     linalg.transpose
+//      CHECK:     tensor.unpack
+
+// -----
+
+#map = affine_map<(d0, d1, d2, d3, d4)->(d1, d2, d0, d4, d3)>
+#map1 = affine_map<(d0, d1, d2, d3, d4)->(d0, d1, d2, d3, d4)>
+func.func @generic_transpose_unpack_fold(%arg0: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> {
+  %0 = tensor.empty() : tensor<5x2x3x16x4xi32>
+  %transposed = linalg.generic {
+                iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"],
+                indexing_maps = [#map, #map1]}
+                ins(%arg0 : tensor<2x3x5x4x16xi32>)
+                outs(%0 : tensor<5x2x3x16x4xi32>) {
+  ^bb0(%in : i32, %out : i32):
+    linalg.yield %in : i32
+  } -> tensor<5x2x3x16x4xi32>
+  %1 = tensor.empty() : tensor<5x48x8xi32>
+  %unpack = tensor.unpack %transposed
+            outer_dims_perm = [0, 2, 1]
+            inner_dims_pos = [1, 2]
+            inner_tiles = [16, 4] into
+            %1 : tensor<5x2x3x16x4xi32> -> tensor<5x48x8xi32>
+  return %unpack : tensor<5x48x8xi32>
+}
+//CHECK-LABEL:  func.func @generic_transpose_unpack_fold(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> {
+//      CHECK:     %[[OUT:.+]] = tensor.empty() : tensor<5x48x8xi32>
+//      CHECK:     %[[UNPACK:.+]] = tensor.unpack %[[ARG0]]
+// CHECK-SAME:        outer_dims_perm = [2, 1, 0]
+// CHECK-SAME:        inner_dims_pos = [2, 1]
+// CHECK-SAME:        inner_tiles = [4, 16]
+// CHEKC-SAME:        into %[[OUT]] : tensor<2x3x5x4x16xi32> -> tensor<5x48x8xi32>
+//      CHECK:     return %[[UNPACK]] : tensor<5x48x8xi32>
+//      CHECK:   }
+
+// -----
+
+#map = affine_map<(d0, d1, d2)->(d1, d2, d0)>
+#map1 = affine_map<(d0, d1, d2)->(d0, d1, d2)>
+func.func @unpack_generic_transpose_fold(%arg0: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> {
+  %0 = tensor.empty() : tensor<3x56x3648xf32>
+  %unpack = tensor.unpack %arg0
+    outer_dims_perm = [2, 0, 1]
+    inner_dims_pos = [1, 2]
+    inner_tiles = [1, 64]
+    into %0 : tensor<57x3x56x1x64xf32> -> tensor<3x56x3648xf32>
+
+  %1 = tensor.empty() : tensor<3648x3x56xf32>
+  %transposed = linalg.generic {
+                iterator_types = ["parallel", "parallel", "parallel"],
+                indexing_maps = [#map, #map1]}
+                ins(%unpack : tensor<3x56x3648xf32>)
+                outs(%1 : tensor<3648x3x56xf32>) {
+  ^bb0(%in : f32, %out : f32):
+    linalg.yield %in : f32
+  } -> tensor<3648x3x56xf32>
+  return %transposed : tensor<3648x3x56xf32>
+}
+// CHECK-LABEL:  func.func @unpack_generic_transpose_fold(
+//  CHECK-SAME:    %[[ARG0:.+]]: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> {
+//       CHECK:        %[[OUT:.+]] = tensor.empty() : tensor<3648x3x56xf32>
+//       CHECK:        %[[UNPACK:.+]] = tensor.unpack %[[ARG0]]
+//  CHECK-SAME:        outer_dims_perm = [0, 1, 2]
+//  CHECK-SAME:        inner_dims_pos = [2, 0]
+//  CHECK-SAME:        inner_tiles = [1, 64]
+//  CHECK-SAME:        into %[[OUT:.+]] : tensor<57x3x56x1x64xf32> -> tensor<3648x3x56xf32>
+//       CHECK:       return %[[UNPACK]] : tensor<3648x3x56xf32>
+//       CHECK:    }
diff --git a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir
index 1a1cf9e407d805d6b07d5de02adda024b3bb794a..35ee6f1caf0d9ba24aad1d5c4a75434a6e4badb5 100644
--- a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir
+++ b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir
@@ -21,3 +21,138 @@ func.func @tensor_generate_constant() -> tensor<2x3x5xf32> {
   } : tensor<2x3x5xf32>
   return %0 : tensor<2x3x5xf32>
 }
+
+//         CHECK-LABEL: func @pad_of_ints(
+//               CHECK: %[[cst:.*]] = arith.constant dense<[
+// CHECK-SAME{LITERAL}:     [0, 0, 0, 0],
+// CHECK-SAME{LITERAL}:     [0, 6, 7, 0],
+// CHECK-SAME{LITERAL}:     [0, 8, 9, 0],
+// CHECK-SAME{LITERAL}:     [0, 0, 0, 0]
+// CHECK-SAME{LITERAL}:     ]> : tensor<4x4xi32>
+//               CHECK: %[[cast:.*]] = tensor.cast %[[cst]] : tensor<4x4xi32> to tensor<?x?xi32>
+//               CHECK: return %[[cast]]
+func.func @pad_of_ints() -> tensor<?x?xi32> {
+  %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32>
+  %pad_value = arith.constant 0 : i32
+
+  %c1 = arith.constant 1 : index
+
+  %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : i32
+  } : tensor<2x2xi32> to tensor<?x?xi32>
+
+  return %0 : tensor<?x?xi32>
+}
+
+//         CHECK-LABEL: func @pad_of_floats(
+//               CHECK: %[[cst:.*]] = arith.constant dense<[
+// CHECK-SAME{LITERAL}:     [0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00],
+// CHECK-SAME{LITERAL}:     [0.000000e+00, 6.000000e+00, 7.000000e+00, 0.000000e+00],
+// CHECK-SAME{LITERAL}:     [0.000000e+00, 8.000000e+00, 9.000000e+00, 0.000000e+00],
+// CHECK-SAME{LITERAL}:     [0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]
+// CHECK-SAME{LITERAL}:     ]> : tensor<4x4xf32>
+//               CHECK: return %[[cst]]
+
+func.func @pad_of_floats() -> tensor<4x4xf32> {
+  %init = arith.constant dense<[[6.0, 7.0], [8.0, 9.0]]> : tensor<2x2xf32>
+  %pad_value = arith.constant 0.0 : f32
+
+  %0 = tensor.pad %init low[1, 1] high[1, 1] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : f32
+  } : tensor<2x2xf32> to tensor<4x4xf32>
+
+  return %0 : tensor<4x4xf32>
+}
+
+//         CHECK-LABEL: func @pad_of_ints_no_low_dims(
+//               CHECK: %[[cst:.*]] = arith.constant dense<[
+// CHECK-SAME{LITERAL}:     [6, 7, 0],
+// CHECK-SAME{LITERAL}:     [8, 9, 0],
+// CHECK-SAME{LITERAL}:     [0, 0, 0]
+// CHECK-SAME{LITERAL}:     ]> : tensor<3x3xi32>
+//               CHECK: return %[[cst]]
+func.func @pad_of_ints_no_low_dims() -> tensor<3x3xi32> {
+  %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32>
+  %pad_value = arith.constant 0 : i32
+
+  %0 = tensor.pad %init low[0, 0] high[1, 1] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : i32
+  } : tensor<2x2xi32> to tensor<3x3xi32>
+
+  return %0 : tensor<3x3xi32>
+}
+
+//         CHECK-LABEL: func @pad_of_ints_no_high_dims(
+//               CHECK: %[[cst:.*]] = arith.constant dense<[
+// CHECK-SAME{LITERAL}:     [0, 0, 0],
+// CHECK-SAME{LITERAL}:     [0, 6, 7],
+// CHECK-SAME{LITERAL}:     [0, 8, 9]
+// CHECK-SAME{LITERAL}:     ]> : tensor<3x3xi32>
+//               CHECK: return %[[cst]]
+func.func @pad_of_ints_no_high_dims() -> tensor<3x3xi32> {
+  %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32>
+  %pad_value = arith.constant 0 : i32
+
+  %0 = tensor.pad %init low[1, 1] high[0, 0] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : i32
+  } : tensor<2x2xi32> to tensor<3x3xi32>
+
+  return %0 : tensor<3x3xi32>
+}
+
+//         CHECK-LABEL: func @pad_multi_use_do_not_fold(
+//               CHECK: %[[pad:.+]] = tensor.pad
+//               CHECK: return %[[pad]]
+func.func @pad_multi_use_do_not_fold() -> (tensor<?x?xi32>, tensor<2x2xi32>) {
+  %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32>
+  %pad_value = arith.constant 0 : i32
+
+  %c1 = arith.constant 1 : index
+
+  %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : i32
+  } : tensor<2x2xi32> to tensor<?x?xi32>
+
+  return %0, %init : tensor<?x?xi32>, tensor<2x2xi32>
+}
+
+// -----
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.tensor.rewrite_as_constant aggressive
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
+//         CHECK-LABEL: func @pad_aggressive_fold(
+//               CHECK: %[[init:.*]] = arith.constant dense<7> : tensor<2x2xi32>
+//               CHECK: %[[cst:.*]] = arith.constant dense<[
+// CHECK-SAME{LITERAL}:     [0, 0, 0, 0],
+// CHECK-SAME{LITERAL}:     [0, 7, 7, 0],
+// CHECK-SAME{LITERAL}:     [0, 7, 7, 0],
+// CHECK-SAME{LITERAL}:     [0, 0, 0, 0]
+// CHECK-SAME{LITERAL}:     ]> : tensor<4x4xi32>
+//               CHECK: %[[cast:.*]] = tensor.cast %[[cst]] : tensor<4x4xi32> to tensor<?x?xi32>
+//               CHECK: return %[[cast]]
+func.func @pad_aggressive_fold() -> (tensor<?x?xi32>, tensor<2x2xi32>) {
+  %init = arith.constant dense<7> : tensor<2x2xi32>
+  %pad_value = arith.constant 0 : i32
+
+  %c1 = arith.constant 1 : index
+
+  %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] {
+    ^bb0(%arg1: index, %arg2: index):
+      tensor.yield %pad_value : i32
+  } : tensor<2x2xi32> to tensor<?x?xi32>
+
+  return %0, %init : tensor<?x?xi32>, tensor<2x2xi32>
+}
diff --git a/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..53f4a8970c7949725d4dc60fcb72683c3bd36805
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir
@@ -0,0 +1,68 @@
+// RUN: mlir-opt %s --transform-interpreter | FileCheck %s
+
+// CHECK-LABEL: @vector_deinterleave_2d
+// CHECK-SAME: %[[SRC:.*]]: vector<2x8xi32>) -> (vector<2x4xi32>, vector<2x4xi32>)
+func.func @vector_deinterleave_2d(%a: vector<2x8xi32>) -> (vector<2x4xi32>, vector<2x4xi32>) {
+  // CHECK: %[[CST:.*]] = arith.constant dense<0>
+  // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0]
+  // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]]
+  // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %[[CST]] [0]
+  // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %[[CST]] [0]
+  // CHECK: %[[SRC_1:.*]] = vector.extract %[[SRC]][1]
+  // CHECK: %[[UNZIP_2:.*]], %[[UNZIP_3:.*]] = vector.deinterleave %[[SRC_1]]
+  // CHECK: %[[RES_2:.*]] = vector.insert %[[UNZIP_2]], %[[RES_0]] [1]
+  // CHECK: %[[RES_3:.*]] = vector.insert %[[UNZIP_3]], %[[RES_1]] [1]
+  // CHECK-NEXT: return %[[RES_2]], %[[RES_3]] : vector<2x4xi32>, vector<2x4xi32>
+  %0, %1 = vector.deinterleave %a : vector<2x8xi32> -> vector<2x4xi32>
+  return %0, %1 : vector<2x4xi32>, vector<2x4xi32>
+}
+
+// CHECK-LABEL: @vector_deinterleave_2d_scalable
+// CHECK-SAME: %[[SRC:.*]]: vector<2x[8]xi32>) -> (vector<2x[4]xi32>, vector<2x[4]xi32>)
+func.func @vector_deinterleave_2d_scalable(%a: vector<2x[8]xi32>) -> (vector<2x[4]xi32>, vector<2x[4]xi32>) {
+  // CHECK: %[[CST:.*]] = arith.constant dense<0>
+  // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0]
+  // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]]
+  // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %[[CST]] [0]
+  // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %[[CST]] [0]
+  // CHECK: %[[SRC_1:.*]] = vector.extract %[[SRC]][1]
+  // CHECK: %[[UNZIP_2:.*]], %[[UNZIP_3:.*]] = vector.deinterleave %[[SRC_1]]
+  // CHECK: %[[RES_2:.*]] = vector.insert %[[UNZIP_2]], %[[RES_0]] [1]
+  // CHECK: %[[RES_3:.*]] = vector.insert %[[UNZIP_3]], %[[RES_1]] [1]
+  // CHECK-NEXT: return %[[RES_2]], %[[RES_3]] : vector<2x[4]xi32>, vector<2x[4]xi32>
+  %0, %1 = vector.deinterleave %a : vector<2x[8]xi32> -> vector<2x[4]xi32>
+  return %0, %1 : vector<2x[4]xi32>, vector<2x[4]xi32>
+}
+
+// CHECK-LABEL: @vector_deinterleave_4d
+// CHECK-SAME: %[[SRC:.*]]: vector<1x2x3x8xi64>) -> (vector<1x2x3x4xi64>, vector<1x2x3x4xi64>)
+func.func @vector_deinterleave_4d(%a: vector<1x2x3x8xi64>) -> (vector<1x2x3x4xi64>, vector<1x2x3x4xi64>) {
+    // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0, 0, 0] : vector<8xi64> from vector<1x2x3x8xi64>
+    // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]] : vector<8xi64> -> vector<4xi64>
+    // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %{{.*}} [0, 0, 0] : vector<4xi64> into vector<1x2x3x4xi64>
+    // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %{{.*}} [0, 0, 0] : vector<4xi64> into vector<1x2x3x4xi64>
+    // CHECK-COUNT-5: vector.deinterleave %{{.*}} : vector<8xi64> -> vector<4xi64>
+    %0, %1 = vector.deinterleave %a : vector<1x2x3x8xi64> -> vector<1x2x3x4xi64>
+    return %0, %1 : vector<1x2x3x4xi64>, vector<1x2x3x4xi64>
+}
+
+// CHECK-LABEL: @vector_deinterleave_nd_with_scalable_dim
+func.func @vector_deinterleave_nd_with_scalable_dim(
+  %a: vector<1x3x[2]x2x3x8xf16>) -> (vector<1x3x[2]x2x3x4xf16>, vector<1x3x[2]x2x3x4xf16>) {
+  // The scalable dim blocks unrolling so only the first two dims are unrolled.
+  // CHECK-COUNT-3: vector.deinterleave %{{.*}} : vector<[2]x2x3x8xf16>
+  %0, %1 = vector.deinterleave %a: vector<1x3x[2]x2x3x8xf16> -> vector<1x3x[2]x2x3x4xf16>
+  return %0, %1 : vector<1x3x[2]x2x3x4xf16>, vector<1x3x[2]x2x3x4xf16>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %f = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+
+    transform.apply_patterns to %f {
+      transform.apply_patterns.vector.lower_interleave
+    } : !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
index 788ae9ac044ed8501cd61125cae9d80ba4e431bb..d7365d25d21b4dbc7e5f7a1fccf977c497b209d5 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
@@ -102,6 +102,25 @@ func.func @transfer_read_dims_mismatch_non_zero_indices(
 
 // -----
 
+func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
+    %subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
+    %idx0 : index, %idx1 : index) -> vector<2x2xf32> {
+  %c0 = arith.constant 0 : index
+  %cst_1 = arith.constant 0.000000e+00 : f32
+  %8 = vector.transfer_read %subview[%c0, %idx0, %idx1, %c0], %cst_1 {in_bounds = [true, true]} : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>, vector<2x2xf32>
+  return %8 : vector<2x2xf32>
+}
+
+//       CHECK:  #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)>
+// CHECK-LABEL:  func.func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
+//       CHECK:    %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>>
+//       CHECK:    %[[APPLY:.*]] = affine.apply #[[$MAP]]()
+
+// CHECK-128B-LABEL: func @transfer_read_dims_mismatch_non_contiguous_non_zero_indices(
+//       CHECK-128B:   memref.collapse_shape
+
+// -----
+
 // The input memref has a dynamic trailing shape and hence is not flattened.
 // TODO: This case could be supported via memref.dim
 
@@ -212,6 +231,25 @@ func.func @transfer_write_dims_mismatch_contiguous(
 
 // -----
 
+func.func @transfer_write_dims_mismatch_non_contiguous_non_zero_indices(
+    %value : vector<2x2xf32>,
+    %subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
+    %idx0 : index, %idx1 : index) {
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %value, %subview[%c0, %idx0, %idx1, %c0] {in_bounds = [true, true]} : vector<2x2xf32>, memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>
+  return
+}
+
+//       CHECK:  #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)>
+// CHECK-LABEL:  func.func @transfer_write_dims_mismatch_non_contiguous_non_zero_indices(
+//       CHECK:    %[[APPLY:.*]] = affine.apply #[[$MAP]]()
+//       CHECK:    %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>>
+
+// CHECK-128B-LABEL: func @transfer_write_dims_mismatch_non_contiguous_non_zero_indices(
+//       CHECK-128B:   memref.collapse_shape
+
+// -----
+
 func.func @transfer_write_dims_mismatch_non_contiguous(
       %arg : memref<5x4x3x2xi8, strided<[24, 6, 2, 1], offset: ?>>, %vec : vector<2x1x2x2xi8>) {
     %c0 = arith.constant 0 : index
@@ -459,37 +497,27 @@ func.func @fold_unit_dims_entirely(%arg0 : vector<8xi32>,
 // CHECK-128B-LABEL: func @fold_unit_dims_entirely(
 //   CHECK-128B-NOT:   memref.collapse_shape
 
-
 // -----
 
-func.func @regression_non_contiguous_dim_read(%subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
-                                              %idx0 : index, %idx1 : index) -> vector<2x2xf32> {
+func.func @negative_out_of_bound_transfer_read(
+    %arg : memref<?x4x3x2xi8, strided<[24, 6, 2, 1], offset: ?>>) -> vector<5x4x3x2xi8> {
   %c0 = arith.constant 0 : index
-  %cst_1 = arith.constant 0.000000e+00 : f32
-  %8 = vector.transfer_read %subview[%c0, %idx0, %idx1, %c0], %cst_1 {in_bounds = [true, true]} : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>, vector<2x2xf32>
-  return %8 : vector<2x2xf32>
+  %cst = arith.constant 0 : i8
+  %v = vector.transfer_read %arg[%c0, %c0, %c0, %c0], %cst {in_bounds = [false, true, true, true]} :
+    memref<?x4x3x2xi8, strided<[24, 6, 2, 1], offset: ?>>, vector<5x4x3x2xi8>
+  return %v : vector<5x4x3x2xi8>
 }
-
-//       CHECK:  #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)>
-// CHECK-LABEL:    func.func @regression_non_contiguous_dim_read(
-//       CHECK:      %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>>
-//       CHECK:     %[[APPLY:.*]] = affine.apply #[[$MAP]]()
-
-// CHECK-128B-LABEL: func @regression_non_contiguous_dim_read(
-//       CHECK-128B:   memref.collapse_shape
+// CHECK:     func.func @negative_out_of_bound_transfer_read
+// CHECK-NOT:   memref.collapse_shape
 
 // -----
 
-func.func @unsupported_non_contiguous_dim_write(%value : vector<2x2xf32>,
-                                                %subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>,
-                                                %idx0 : index, %idx1 : index) {
+func.func @negative_out_of_bound_transfer_write(
+    %arg : memref<?x4x3x2xi8, strided<[24, 6, 2, 1], offset: ?>>, %vec : vector<1x1x3x2xi8>) {
   %c0 = arith.constant 0 : index
-  vector.transfer_write %value, %subview[%c0, %idx0, %idx1, %c0] {in_bounds = [true, true]} : vector<2x2xf32>, memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>
+  vector.transfer_write %vec, %arg [%c0, %c0, %c0, %c0] {in_bounds = [false, true, true, true]} :
+    vector<1x1x3x2xi8>, memref<?x4x3x2xi8, strided<[24, 6, 2, 1], offset: ?>>
   return
 }
-
-// CHECK-LABEL:  func.func @unsupported_non_contiguous_dim_write(
-//   CHECK-NOT:    memref.collapse_shape
-
-// CHECK-128B-LABEL: func @unsupported_non_contiguous_dim_write(
-//   CHECK-128B-NOT:   memref.collapse_shape
+// CHECK:     func.func @negative_out_of_bound_transfer_write
+// CHECK-NOT:   memref.collapse_shape
diff --git a/mlir/test/IR/array-of-attr.mlir b/mlir/test/IR/array-of-attr.mlir
index 1b6fe55205959e2c0f30f9b1061bf350eba639f0..c2a60759658269f828deada05984f3e25b6985ff 100644
--- a/mlir/test/IR/array-of-attr.mlir
+++ b/mlir/test/IR/array-of-attr.mlir
@@ -12,3 +12,7 @@ test.array_of_attr_op
 // CHECK: test.array_of_attr_op
 // CHECK-SAME: a = [], b = [], c = []
 test.array_of_attr_op a = [], b = [], c = []
+
+// CHECK: "test.test_array_float"
+// CHECK-SAME: 1.000000e+00 : f32, 1.000000e+00, 0x7FF0000000000000 : f64
+"test.test_array_float"() {test.float_arr = [1.0 : f32, 1.0 : f64, 0x7FF0000000000000 : f64]} : () -> ()
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
index 10ffed2688178c5a5d3e73396e00dd2f6df13a7c..aabd9d2ce788e2254a500ec5d0a0824c130d2a4b 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
@@ -4,7 +4,7 @@
 // RUN:   -arm-sme-vector-legalization -canonicalize -cse \
 // RUN:   -convert-vector-to-arm-sme -arm-sme-outer-product-fusion \
 // RUN:   -allocate-arm-sme-tiles -convert-arm-sme-to-scf \
-// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za only-if-required-by-ops" \
+// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za if-required-by-ops" \
 // RUN:   -convert-vector-to-scf=full-unroll -convert-arm-sme-to-llvm \
 // RUN:   -test-lower-to-llvm | \
 // RUN: %mcr_aarch64_cmd \
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg
deleted file mode 100644
index 0d8ad605f598fb0206fd1c24b58528536a82d859..0000000000000000000000000000000000000000
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-# The tests in this folder assume full control of the hardware features, such as
-# the vector length, so must be run under an emulator.
-
-if not config.arm_emulator_executable:
-    config.unsupported = True
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
similarity index 98%
rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir
rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
index 0648e771b88913da7a9325ccd7e16b37af899909..4c02bcb9d192fb9b6008d287114bec01ae67277b 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
@@ -1,3 +1,5 @@
+// REQUIRES: arm-emulator
+
 // DEFINE: %{entry_point} = main
 // DEFINE: %{compile} = mlir-opt %s \
 // DEFINE: --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm),test-lower-to-llvm)"
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg
deleted file mode 100644
index 0d8ad605f598fb0206fd1c24b58528536a82d859..0000000000000000000000000000000000000000
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-# The tests in this folder assume full control of the hardware features, such as
-# the vector length, so must be run under an emulator.
-
-if not config.arm_emulator_executable:
-    config.unsupported = True
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
similarity index 97%
rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir
rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
index e6c561437132f613e8214f33677b066a392ebcdb..3ef41b0bf5cf453e62f11b5a98dd14902d1044b4 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
@@ -1,3 +1,5 @@
+// REQUIRES: arm-emulator
+
 // DEFINE: %{entry_point} = entry
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
similarity index 97%
rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir
rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
index aa8d0e4d5104abf007993fc3fd3b0a8574e7944c..6e25d9d16b8bde50290e23133b8b85f93c3d7443 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
@@ -1,3 +1,5 @@
+// REQUIRES: arm-emulator
+
 // DEFINE: %{entry_point} = main
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
diff --git a/mlir/test/Target/Cpp/for.mlir b/mlir/test/Target/Cpp/for.mlir
index 60988bcb4655621992a38783fee68275c347b102..af1d829113f9dd81068cc48525692fac13d9c5ca 100644
--- a/mlir/test/Target/Cpp/for.mlir
+++ b/mlir/test/Target/Cpp/for.mlir
@@ -40,8 +40,6 @@ func.func @test_for_yield() {
   %s0 = "emitc.constant"() <{value = 0 : i32}> : () -> i32
   %p0 = "emitc.constant"() <{value = 1.0 : f32}> : () -> f32
 
-  %0 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
-  %1 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
   %2 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
   %3 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
   emitc.assign %s0 : i32 to %2 : i32
@@ -53,8 +51,6 @@ func.func @test_for_yield() {
     emitc.assign %pn : f32 to %3 : f32
     emitc.yield
   }
-  emitc.assign %2 : i32 to %0 : i32
-  emitc.assign %3 : f32 to %1 : f32
 
   return
 }
@@ -64,8 +60,6 @@ func.func @test_for_yield() {
 // CPP-DEFAULT-NEXT: size_t [[STEP:[^ ]*]] = 1;
 // CPP-DEFAULT-NEXT: int32_t [[S0:[^ ]*]] = 0;
 // CPP-DEFAULT-NEXT: float [[P0:[^ ]*]] = 1.000000000e+00f;
-// CPP-DEFAULT-NEXT: int32_t [[SE:[^ ]*]];
-// CPP-DEFAULT-NEXT: float [[PE:[^ ]*]];
 // CPP-DEFAULT-NEXT: int32_t [[SI:[^ ]*]];
 // CPP-DEFAULT-NEXT: float [[PI:[^ ]*]];
 // CPP-DEFAULT-NEXT: [[SI:[^ ]*]] = [[S0]];
@@ -76,8 +70,6 @@ func.func @test_for_yield() {
 // CPP-DEFAULT-NEXT: [[SI]] = [[SN]];
 // CPP-DEFAULT-NEXT: [[PI]] = [[PN]];
 // CPP-DEFAULT-NEXT: }
-// CPP-DEFAULT-NEXT: [[SE]] = [[SI]];
-// CPP-DEFAULT-NEXT: [[PE]] = [[PI]];
 // CPP-DEFAULT-NEXT: return;
 
 // CPP-DECLTOP: void test_for_yield() {
@@ -86,8 +78,6 @@ func.func @test_for_yield() {
 // CPP-DECLTOP-NEXT: size_t [[STEP:[^ ]*]];
 // CPP-DECLTOP-NEXT: int32_t [[S0:[^ ]*]];
 // CPP-DECLTOP-NEXT: float [[P0:[^ ]*]];
-// CPP-DECLTOP-NEXT: int32_t [[SE:[^ ]*]];
-// CPP-DECLTOP-NEXT: float [[PE:[^ ]*]];
 // CPP-DECLTOP-NEXT: int32_t [[SI:[^ ]*]];
 // CPP-DECLTOP-NEXT: float [[PI:[^ ]*]];
 // CPP-DECLTOP-NEXT: int32_t [[SN:[^ ]*]];
@@ -99,8 +89,6 @@ func.func @test_for_yield() {
 // CPP-DECLTOP-NEXT: [[P0]] = 1.000000000e+00f;
 // CPP-DECLTOP-NEXT: ;
 // CPP-DECLTOP-NEXT: ;
-// CPP-DECLTOP-NEXT: ;
-// CPP-DECLTOP-NEXT: ;
 // CPP-DECLTOP-NEXT: [[SI:[^ ]*]] = [[S0]];
 // CPP-DECLTOP-NEXT: [[PI:[^ ]*]] = [[P0]];
 // CPP-DECLTOP-NEXT: for (size_t [[ITER:[^ ]*]] = [[START]]; [[ITER]] < [[STOP]]; [[ITER]] += [[STEP]]) {
@@ -109,8 +97,6 @@ func.func @test_for_yield() {
 // CPP-DECLTOP-NEXT: [[SI]] = [[SN]];
 // CPP-DECLTOP-NEXT: [[PI]] = [[PN]];
 // CPP-DECLTOP-NEXT: }
-// CPP-DECLTOP-NEXT: [[SE]] = [[SI]];
-// CPP-DECLTOP-NEXT: [[PE]] = [[PI]];
 // CPP-DECLTOP-NEXT: return;
 
 func.func @test_for_yield_2() {
@@ -121,8 +107,6 @@ func.func @test_for_yield_2() {
   %s0 = emitc.literal "0" : i32
   %p0 = emitc.literal "M_PI" : f32
 
-  %0 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
-  %1 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
   %2 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
   %3 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
   emitc.assign %s0 : i32 to %2 : i32
@@ -134,8 +118,6 @@ func.func @test_for_yield_2() {
     emitc.assign %pn : f32 to %3 : f32
     emitc.yield
   }
-  emitc.assign %2 : i32 to %0 : i32
-  emitc.assign %3 : f32 to %1 : f32
 
   return
 }
diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll
index 0c6a7368b7b88bcb6ce8fab1a799790e19473c83..2173cc33e68a4f350b5a1561cba6f363b0f920ce 100644
--- a/mlir/test/Target/LLVMIR/Import/debug-info.ll
+++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll
@@ -761,3 +761,35 @@ define void @class_field(ptr %arg1) !dbg !18 {
 !11 = !{!6, !7, !8} ; C -> A, B, C
 
 !18 = distinct !DISubprogram(name: "SP", scope: !3, file: !2, spFlags: DISPFlagDefinition, unit: !1)
+
+; // -----
+
+; Verify the string type is handled correctly
+
+define void @string_type(ptr %arg1) {
+  call void @llvm.dbg.value(metadata ptr %arg1, metadata !4, metadata !DIExpression()), !dbg !10
+  call void @llvm.dbg.value(metadata ptr %arg1, metadata !9, metadata !DIExpression()), !dbg !10
+  ret void
+}
+
+!llvm.dbg.cu = !{!1}
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2)
+!2 = !DIFile(filename: "debug-info.ll", directory: "/")
+!3 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!4 = !DILocalVariable(scope: !5, name: "string_size", file: !2, type: !3);
+!5 = distinct !DISubprogram(name: "test", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1)
+!6 = !DIStringType(name: "character(*)", stringLength: !4, size: 32, align: 8, stringLengthExpression: !8, stringLocationExpression: !7)
+!7 = !DIExpression(DW_OP_push_object_address, DW_OP_deref)
+!8 = !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8)
+!9 = !DILocalVariable(scope: !5, name: "str", file: !2, type: !6);
+!10 = !DILocation(line: 1, column: 2, scope: !5)
+
+; CHECK: #[[VAR:.+]] = #llvm.di_local_variable<{{.*}}name = "string_size"{{.*}}>
+; CHECK: #llvm.di_string_type<tag = DW_TAG_string_type, name = "character(*)"
+; CHECK-SAME: sizeInBits = 32
+; CHECK-SAME: alignInBits = 8
+; CHECK-SAME: stringLength = #[[VAR]]
+; CHECK-SAME: stringLengthExp = <[DW_OP_push_object_address, DW_OP_plus_uconst(8)]>
+; CHECK-SAME: stringLocationExp = <[DW_OP_push_object_address, DW_OP_deref]>>
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index 724f45f0d2a876322d0466dc5044a081e30851ea..c6e7ca6f3f21d0446e0fd387f02027b558600c7a 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -564,3 +564,30 @@ llvm.func @subranges(%arg: !llvm.ptr) {
 // CHECK: ![[ELEMENTS2]] = !{![[ELEMENT2:[0-9]+]]}
 // CHECK: ![[ELEMENT2]] = !DISubrange(count: ![[LV:[0-9]+]], stride: ![[GV:[0-9]+]])
 // CHECK: ![[LV]] = !DILocalVariable(name: "size"{{.*}})
+
+// -----
+
+#bt = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "int", sizeInBits = 32>
+#file = #llvm.di_file<"debug-info.ll" in "/">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_C,
+ file = #file, isOptimized = false, emissionKind = Full>
+#sp = #llvm.di_subprogram<compileUnit = #cu, scope = #file, name = "test",
+ file = #file, subprogramFlags = Definition>
+#var = #llvm.di_local_variable<scope = #sp, name = "string_size", type = #bt>
+#ty = #llvm.di_string_type<tag = DW_TAG_string_type, name = "character(*)",
+ sizeInBits = 32, alignInBits = 8, stringLength = #var,
+ stringLengthExp = <[DW_OP_push_object_address, DW_OP_plus_uconst(8)]>,
+ stringLocationExp = <[DW_OP_push_object_address, DW_OP_deref]>>
+#var1 = #llvm.di_local_variable<scope = #sp, name = "str", type = #ty>
+
+llvm.func @string_ty(%arg0: !llvm.ptr) {
+  llvm.intr.dbg.value #var1 = %arg0 : !llvm.ptr
+  llvm.intr.dbg.value #var = %arg0 : !llvm.ptr
+  llvm.return
+} loc(#loc2)
+
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc(fused<#sp>[#loc1])
+
+// CHECK-DAG: !DIStringType(name: "character(*)", stringLength: ![[VAR:[0-9]+]], stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8), stringLocationExpression: !DIExpression(DW_OP_push_object_address, DW_OP_deref), size: 32, align: 8)
+// CHECK-DAG: ![[VAR]] = !DILocalVariable(name: "string_size"{{.*}})
diff --git a/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp b/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
index d3dabaf200fdc2acde60d8c6924b8e532937cfd4..a220791969d53bb22e62beff51412cdf50152b7e 100644
--- a/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
+++ b/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
@@ -74,7 +74,7 @@ void buildTestLowerToArmSME(OpPassManager &pm,
   // Enable streaming-mode and ZA.
   pm.addPass(arm_sme::createEnableArmStreamingPass(
       arm_sme::ArmStreamingMode::StreamingLocally, arm_sme::ArmZaMode::NewZA,
-      /*onlyIfRequiredByOps=*/true));
+      /*ifRequiredByOps=*/true));
 
   // Convert SCF to CF (required for ArmSME tile allocation).
   pm.addPass(createConvertSCFToCFPass());
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index 9ed3a2efcb8f71133f67581b17c7d35cf4fd7cb0..9838a02136d143d6b4506a855059e6cbd04ba172 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -250,3 +250,6 @@ if config.run_nvptx_tests:
 
 if config.run_rocm_tests:
     config.available_features.add("host-supports-amdgpu")
+
+if config.arm_emulator_executable:
+    config.available_features.add("arm-emulator")
diff --git a/mlir/test/python/ir/affine_expr.py b/mlir/test/python/ir/affine_expr.py
index 63564303e8315f9b45346acc3b3f6e5dc0074d5e..c7861c1acfe1252a70ddf6265fec47b0fc96452b 100644
--- a/mlir/test/python/ir/affine_expr.py
+++ b/mlir/test/python/ir/affine_expr.py
@@ -42,7 +42,7 @@ def testAffineExprEq():
         # CHECK: False
         print(a1 == a3)
         # CHECK: False
-        print(a1 == None)
+        print(a1 is None)
         # CHECK: False
         print(a1 == "foo")
 
diff --git a/mlir/test/python/ir/attributes.py b/mlir/test/python/ir/attributes.py
index dbd6bad05e01dc2cb48d4e131e5491c861136160..0f2c8e7b7252ace0b170ca7466aa4f0002d4464f 100644
--- a/mlir/test/python/ir/attributes.py
+++ b/mlir/test/python/ir/attributes.py
@@ -56,8 +56,8 @@ def testAttrEq():
         print("a1 == a2:", a1 == a2)
         # CHECK: a1 == a3: True
         print("a1 == a3:", a1 == a3)
-        # CHECK: a1 == None: False
-        print("a1 == None:", a1 == None)
+        # CHECK: a1 is None: False
+        print("a1 is None:", a1 is None)
 
 
 # CHECK-LABEL: TEST: testAttrHash
@@ -109,9 +109,9 @@ def testAttrEqDoesNotRaise():
         # CHECK: False
         print(a1 == not_an_attr)
         # CHECK: False
-        print(a1 == None)
+        print(a1 is None)
         # CHECK: True
-        print(a1 != None)
+        print(a1 is not None)
 
 
 # CHECK-LABEL: TEST: testAttrCapsule
diff --git a/mlir/test/python/ir/blocks.py b/mlir/test/python/ir/blocks.py
index 8b4d946c97b8ddd51382c06265ff310c3eac7715..70ccaeeb5435bb3956dcf6c58931087df3137e22 100644
--- a/mlir/test/python/ir/blocks.py
+++ b/mlir/test/python/ir/blocks.py
@@ -145,3 +145,35 @@ def testBlockHash():
             block1 = Block.create_at_start(dummy.operation.regions[0], [f32])
             block2 = Block.create_at_start(dummy.operation.regions[0], [f32])
             assert hash(block1) != hash(block2)
+
+
+# CHECK-LABEL: TEST: testBlockAddArgs
+@run
+def testBlockAddArgs():
+    with Context() as ctx, Location.unknown(ctx) as loc:
+        ctx.allow_unregistered_dialects = True
+        f32 = F32Type.get()
+        op = Operation.create("test", regions=1, loc=Location.unknown())
+        blocks = op.regions[0].blocks
+        blocks.append()
+        # CHECK: ^bb0:
+        op.print(enable_debug_info=True)
+        blocks[0].add_argument(f32, loc)
+        # CHECK: ^bb0(%{{.+}}: f32 loc(unknown)):
+        op.print(enable_debug_info=True)
+
+
+# CHECK-LABEL: TEST: testBlockEraseArgs
+@run
+def testBlockEraseArgs():
+    with Context() as ctx, Location.unknown(ctx) as loc:
+        ctx.allow_unregistered_dialects = True
+        f32 = F32Type.get()
+        op = Operation.create("test", regions=1, loc=Location.unknown())
+        blocks = op.regions[0].blocks
+        blocks.append(f32)
+        # CHECK: ^bb0(%{{.+}}: f32 loc(unknown)):
+        op.print(enable_debug_info=True)
+        blocks[0].erase_argument(0)
+        # CHECK: ^bb0:
+        op.print(enable_debug_info=True)
diff --git a/mlir/test/python/ir/builtin_types.py b/mlir/test/python/ir/builtin_types.py
index 4eea1a9c372ef7feb3ab4e38465fb5f18c8bd3ea..cfe377c70371716ed27fdf49d824943a454d86e7 100644
--- a/mlir/test/python/ir/builtin_types.py
+++ b/mlir/test/python/ir/builtin_types.py
@@ -57,8 +57,8 @@ def testTypeEq():
     print("t1 == t2:", t1 == t2)
     # CHECK: t1 == t3: True
     print("t1 == t3:", t1 == t3)
-    # CHECK: t1 == None: False
-    print("t1 == None:", t1 == None)
+    # CHECK: t1 is None: False
+    print("t1 is None:", t1 is None)
 
 
 # CHECK-LABEL: TEST: testTypeHash
@@ -143,9 +143,9 @@ def testTypeEqDoesNotRaise():
     # CHECK: False
     print(t1 == not_a_type)
     # CHECK: False
-    print(t1 == None)
+    print(t1 is None)
     # CHECK: True
-    print(t1 != None)
+    print(t1 is not None)
 
 
 # CHECK-LABEL: TEST: testTypeCapsule
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 0e8b161d513456800b9ff7f485418603d901b89a..d2ba3d06835fb18dae0428cffc1031b6155c1442 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -32,21 +32,21 @@ using namespace mlir;
 
 // Defined in the test directory, no public header.
 namespace mlir {
-void registerConvertToTargetEnvPass();
 void registerCloneTestPasses();
+void registerConvertToTargetEnvPass();
 void registerLazyLoadingTestPasses();
+void registerLoopLikeInterfaceTestPasses();
 void registerPassManagerTestPass();
 void registerPrintSpirvAvailabilityPass();
-void registerLoopLikeInterfaceTestPasses();
+void registerRegionTestPasses();
 void registerShapeFunctionTestPasses();
 void registerSideEffectTestPasses();
 void registerSliceAnalysisTestPass();
 void registerSymbolTestPasses();
-void registerRegionTestPasses();
-void registerTestAffineDataCopyPass();
 void registerTestAffineAccessAnalysisPass();
-void registerTestAffineReifyValueBoundsPass();
+void registerTestAffineDataCopyPass();
 void registerTestAffineLoopUnswitchingPass();
+void registerTestAffineReifyValueBoundsPass();
 void registerTestAffineWalk();
 void registerTestBytecodeRoundtripPasses();
 void registerTestDecomposeAffineOpPass();
@@ -56,10 +56,10 @@ void registerTestGpuMemoryPromotionPass();
 void registerTestLoopPermutationPass();
 void registerTestMatchers();
 void registerTestOperationEqualPass();
+void registerTestPreserveUseListOrders();
 void registerTestPrintDefUsePass();
 void registerTestPrintInvalidPass();
 void registerTestPrintNestingPass();
-void registerTestPreserveUseListOrders();
 void registerTestReducer();
 void registerTestSpirvEntryPointABIPass();
 void registerTestSpirvModuleCombinerPass();
@@ -68,7 +68,6 @@ void registerTosaTestQuantUtilAPIPass();
 void registerVectorizerTestPass();
 
 namespace test {
-void registerTestCompositePass();
 void registerCommutativityUtils();
 void registerConvertCallOpPass();
 void registerConvertFuncOpPass();
@@ -77,12 +76,15 @@ void registerMemRefBoundCheck();
 void registerPatternsTestPass();
 void registerSimpleParametricTilingPass();
 void registerTestAffineLoopParametricTilingPass();
-void registerTestArithEmulateWideIntPass();
 void registerTestAliasAnalysisPass();
+void registerTestArithEmulateWideIntPass();
 void registerTestBuiltinAttributeInterfaces();
 void registerTestBuiltinDistinctAttributes();
 void registerTestCallGraphPass();
 void registerTestCfAssertPass();
+void registerTestCFGLoopInfoPass();
+void registerTestComposeSubView();
+void registerTestCompositePass();
 void registerTestConstantFold();
 void registerTestControlFlowSink();
 void registerTestDataLayoutPropagation();
@@ -95,12 +97,10 @@ void registerTestDynamicPipelinePass();
 void registerTestEmulateNarrowTypePass();
 void registerTestExpandMathPass();
 void registerTestFooAnalysisPass();
-void registerTestComposeSubView();
-void registerTestMultiBuffering();
-void registerTestIntRangeInference();
-void registerTestIRVisitorsPass();
 void registerTestGenericIRVisitorsPass();
 void registerTestInterfaces();
+void registerTestIntRangeInference();
+void registerTestIRVisitorsPass();
 void registerTestLastModifiedPass();
 void registerTestLinalgDecomposeOps();
 void registerTestLinalgDropUnitDims();
@@ -110,7 +110,6 @@ void registerTestLinalgTransforms();
 void registerTestLivenessAnalysisPass();
 void registerTestLivenessPass();
 void registerTestLoopFusion();
-void registerTestCFGLoopInfoPass();
 void registerTestLoopMappingPass();
 void registerTestLoopUnrollingPass();
 void registerTestLowerToArmNeon();
@@ -123,12 +122,14 @@ void registerTestMathPolynomialApproximationPass();
 void registerTestMathToVCIXPass();
 void registerTestMemRefDependenceCheck();
 void registerTestMemRefStrideCalculation();
-void registerTestMeshSimplificationsPass();
 void registerTestMeshReshardingSpmdizationPass();
-void registerTestOpLoweringPasses();
+void registerTestMeshSimplificationsPass();
+void registerTestMultiBuffering();
 void registerTestNextAccessPass();
+void registerTestNVGPULowerings();
 void registerTestOneToNTypeConversionPass();
 void registerTestOpaqueLoc();
+void registerTestOpLoweringPasses();
 void registerTestPadFusion();
 void registerTestRecursiveTypesPass();
 void registerTestSCFUpliftWhileToFor();
@@ -141,10 +142,9 @@ void registerTestTensorCopyInsertionPass();
 void registerTestTensorTransforms();
 void registerTestTopologicalSortAnalysisPass();
 void registerTestTransformDialectEraseSchedulePass();
-void registerTestWrittenToPass();
 void registerTestVectorLowerings();
 void registerTestVectorReductionToSPIRVDotProd();
-void registerTestNVGPULowerings();
+void registerTestWrittenToPass();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
 void registerTestDialectConversionPasses();
 void registerTestPDLByteCodePass();
@@ -164,17 +164,17 @@ void registerTestTransformDialectExtension(DialectRegistry &);
 void registerTestPasses() {
   registerCloneTestPasses();
   registerConvertToTargetEnvPass();
-  registerPassManagerTestPass();
-  registerPrintSpirvAvailabilityPass();
   registerLazyLoadingTestPasses();
   registerLoopLikeInterfaceTestPasses();
+  registerPassManagerTestPass();
+  registerPrintSpirvAvailabilityPass();
+  registerRegionTestPasses();
   registerShapeFunctionTestPasses();
   registerSideEffectTestPasses();
   registerSliceAnalysisTestPass();
   registerSymbolTestPasses();
-  registerRegionTestPasses();
-  registerTestAffineDataCopyPass();
   registerTestAffineAccessAnalysisPass();
+  registerTestAffineDataCopyPass();
   registerTestAffineLoopUnswitchingPass();
   registerTestAffineReifyValueBoundsPass();
   registerTestAffineWalk();
@@ -186,18 +186,17 @@ void registerTestPasses() {
   registerTestLoopPermutationPass();
   registerTestMatchers();
   registerTestOperationEqualPass();
+  registerTestPreserveUseListOrders();
   registerTestPrintDefUsePass();
   registerTestPrintInvalidPass();
   registerTestPrintNestingPass();
-  registerTestPreserveUseListOrders();
   registerTestReducer();
   registerTestSpirvEntryPointABIPass();
   registerTestSpirvModuleCombinerPass();
   registerTestTraitsPass();
-  registerVectorizerTestPass();
   registerTosaTestQuantUtilAPIPass();
+  registerVectorizerTestPass();
 
-  mlir::test::registerTestCompositePass();
   mlir::test::registerCommutativityUtils();
   mlir::test::registerConvertCallOpPass();
   mlir::test::registerConvertFuncOpPass();
@@ -212,24 +211,25 @@ void registerTestPasses() {
   mlir::test::registerTestBuiltinDistinctAttributes();
   mlir::test::registerTestCallGraphPass();
   mlir::test::registerTestCfAssertPass();
+  mlir::test::registerTestCFGLoopInfoPass();
+  mlir::test::registerTestComposeSubView();
+  mlir::test::registerTestCompositePass();
   mlir::test::registerTestConstantFold();
   mlir::test::registerTestControlFlowSink();
-  mlir::test::registerTestDiagnosticsPass();
-  mlir::test::registerTestDecomposeCallGraphTypes();
   mlir::test::registerTestDataLayoutPropagation();
   mlir::test::registerTestDataLayoutQuery();
   mlir::test::registerTestDeadCodeAnalysisPass();
+  mlir::test::registerTestDecomposeCallGraphTypes();
+  mlir::test::registerTestDiagnosticsPass();
   mlir::test::registerTestDominancePass();
   mlir::test::registerTestDynamicPipelinePass();
   mlir::test::registerTestEmulateNarrowTypePass();
   mlir::test::registerTestExpandMathPass();
   mlir::test::registerTestFooAnalysisPass();
-  mlir::test::registerTestComposeSubView();
-  mlir::test::registerTestMultiBuffering();
-  mlir::test::registerTestIntRangeInference();
-  mlir::test::registerTestIRVisitorsPass();
   mlir::test::registerTestGenericIRVisitorsPass();
   mlir::test::registerTestInterfaces();
+  mlir::test::registerTestIntRangeInference();
+  mlir::test::registerTestIRVisitorsPass();
   mlir::test::registerTestLastModifiedPass();
   mlir::test::registerTestLinalgDecomposeOps();
   mlir::test::registerTestLinalgDropUnitDims();
@@ -239,7 +239,6 @@ void registerTestPasses() {
   mlir::test::registerTestLivenessAnalysisPass();
   mlir::test::registerTestLivenessPass();
   mlir::test::registerTestLoopFusion();
-  mlir::test::registerTestCFGLoopInfoPass();
   mlir::test::registerTestLoopMappingPass();
   mlir::test::registerTestLoopUnrollingPass();
   mlir::test::registerTestLowerToArmNeon();
@@ -252,12 +251,14 @@ void registerTestPasses() {
   mlir::test::registerTestMathToVCIXPass();
   mlir::test::registerTestMemRefDependenceCheck();
   mlir::test::registerTestMemRefStrideCalculation();
-  mlir::test::registerTestOpLoweringPasses();
-  mlir::test::registerTestMeshSimplificationsPass();
   mlir::test::registerTestMeshReshardingSpmdizationPass();
+  mlir::test::registerTestMeshSimplificationsPass();
+  mlir::test::registerTestMultiBuffering();
   mlir::test::registerTestNextAccessPass();
+  mlir::test::registerTestNVGPULowerings();
   mlir::test::registerTestOneToNTypeConversionPass();
   mlir::test::registerTestOpaqueLoc();
+  mlir::test::registerTestOpLoweringPasses();
   mlir::test::registerTestPadFusion();
   mlir::test::registerTestRecursiveTypesPass();
   mlir::test::registerTestSCFUpliftWhileToFor();
@@ -272,7 +273,6 @@ void registerTestPasses() {
   mlir::test::registerTestTransformDialectEraseSchedulePass();
   mlir::test::registerTestVectorLowerings();
   mlir::test::registerTestVectorReductionToSPIRVDotProd();
-  mlir::test::registerTestNVGPULowerings();
   mlir::test::registerTestWrittenToPass();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
   mlir::test::registerTestDialectConversionPasses();
diff --git a/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp b/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
index 6bc0fd6113b9bbf00c633d799719879024e39ca6..53a4af14d119a467ff073049144898c9487746d1 100644
--- a/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
+++ b/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
@@ -27,27 +27,65 @@ protected:
   }
 
   void checkUnidimensional(LoopLikeOpInterface loopLikeOp) {
-    std::optional<OpFoldResult> maybeLb = loopLikeOp.getSingleLowerBound();
-    EXPECT_TRUE(maybeLb.has_value());
-    std::optional<OpFoldResult> maybeUb = loopLikeOp.getSingleUpperBound();
-    EXPECT_TRUE(maybeUb.has_value());
-    std::optional<OpFoldResult> maybeStep = loopLikeOp.getSingleStep();
-    EXPECT_TRUE(maybeStep.has_value());
-    std::optional<OpFoldResult> maybeIndVar =
+    std::optional<OpFoldResult> maybeSingleLb =
+        loopLikeOp.getSingleLowerBound();
+    EXPECT_TRUE(maybeSingleLb.has_value());
+    std::optional<OpFoldResult> maybeSingleUb =
+        loopLikeOp.getSingleUpperBound();
+    EXPECT_TRUE(maybeSingleUb.has_value());
+    std::optional<OpFoldResult> maybeSingleStep = loopLikeOp.getSingleStep();
+    EXPECT_TRUE(maybeSingleStep.has_value());
+    std::optional<OpFoldResult> maybeSingleIndVar =
         loopLikeOp.getSingleInductionVar();
-    EXPECT_TRUE(maybeIndVar.has_value());
+    EXPECT_TRUE(maybeSingleIndVar.has_value());
+
+    std::optional<SmallVector<OpFoldResult>> maybeLb =
+        loopLikeOp.getLoopLowerBounds();
+    ASSERT_TRUE(maybeLb.has_value());
+    EXPECT_EQ((*maybeLb).size(), 1u);
+    std::optional<SmallVector<OpFoldResult>> maybeUb =
+        loopLikeOp.getLoopUpperBounds();
+    ASSERT_TRUE(maybeUb.has_value());
+    EXPECT_EQ((*maybeUb).size(), 1u);
+    std::optional<SmallVector<OpFoldResult>> maybeStep =
+        loopLikeOp.getLoopSteps();
+    ASSERT_TRUE(maybeStep.has_value());
+    EXPECT_EQ((*maybeStep).size(), 1u);
+    std::optional<SmallVector<Value>> maybeInductionVars =
+        loopLikeOp.getLoopInductionVars();
+    ASSERT_TRUE(maybeInductionVars.has_value());
+    EXPECT_EQ((*maybeInductionVars).size(), 1u);
   }
 
   void checkMultidimensional(LoopLikeOpInterface loopLikeOp) {
-    std::optional<OpFoldResult> maybeLb = loopLikeOp.getSingleLowerBound();
-    EXPECT_FALSE(maybeLb.has_value());
-    std::optional<OpFoldResult> maybeUb = loopLikeOp.getSingleUpperBound();
-    EXPECT_FALSE(maybeUb.has_value());
-    std::optional<OpFoldResult> maybeStep = loopLikeOp.getSingleStep();
-    EXPECT_FALSE(maybeStep.has_value());
-    std::optional<OpFoldResult> maybeIndVar =
+    std::optional<OpFoldResult> maybeSingleLb =
+        loopLikeOp.getSingleLowerBound();
+    EXPECT_FALSE(maybeSingleLb.has_value());
+    std::optional<OpFoldResult> maybeSingleUb =
+        loopLikeOp.getSingleUpperBound();
+    EXPECT_FALSE(maybeSingleUb.has_value());
+    std::optional<OpFoldResult> maybeSingleStep = loopLikeOp.getSingleStep();
+    EXPECT_FALSE(maybeSingleStep.has_value());
+    std::optional<OpFoldResult> maybeSingleIndVar =
         loopLikeOp.getSingleInductionVar();
-    EXPECT_FALSE(maybeIndVar.has_value());
+    EXPECT_FALSE(maybeSingleIndVar.has_value());
+
+    std::optional<SmallVector<OpFoldResult>> maybeLb =
+        loopLikeOp.getLoopLowerBounds();
+    ASSERT_TRUE(maybeLb.has_value());
+    EXPECT_EQ((*maybeLb).size(), 2u);
+    std::optional<SmallVector<OpFoldResult>> maybeUb =
+        loopLikeOp.getLoopUpperBounds();
+    ASSERT_TRUE(maybeUb.has_value());
+    EXPECT_EQ((*maybeUb).size(), 2u);
+    std::optional<SmallVector<OpFoldResult>> maybeStep =
+        loopLikeOp.getLoopSteps();
+    ASSERT_TRUE(maybeStep.has_value());
+    EXPECT_EQ((*maybeStep).size(), 2u);
+    std::optional<SmallVector<Value>> maybeInductionVars =
+        loopLikeOp.getLoopInductionVars();
+    ASSERT_TRUE(maybeInductionVars.has_value());
+    EXPECT_EQ((*maybeInductionVars).size(), 2u);
   }
 
   MLIRContext context;
diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 03b620186562024c5e268a882d8ffccc68fd750a..ead2aed414ffe7af1ab7fb453f331a2c5e826a26 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -7,6 +7,8 @@ set(LLVM_SUBPROJECT_TITLE "liboffload")
 if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   set(OPENMP_STANDALONE_BUILD TRUE)
   project(offload C CXX ASM)
+else()
+  set(OPENMP_STANDALONE_BUILD FALSE)
 endif()
 
 # Check that the library can actually be built.
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index b356003672760b4b5ce9b0d76654b5e0c4668127..d88430a52b8b73321d2a46ec90a8d5ab2e80aff2 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -46,7 +46,7 @@ set(all_amdgpu_architectures "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906"
                              "gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1010"
                              "gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035"
                              "gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150"
-                             "gfx1151")
+                             "gfx1151;gfx1152")
 set(all_nvptx_architectures "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62"
                             "sm_70;sm_72;sm_75;sm_80;sm_86;sm_87;sm_89;sm_90")
 set(all_gpu_architectures
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 1d6804da75d928039c90866b7907187d2b9ea85b..fce2adca49c11c9afc041b5b5181ff4dbf505d84 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -64,10 +64,6 @@ struct PluginManager {
         std::make_unique<DeviceImageTy>(TgtBinDesc, TgtDeviceImage));
   }
 
-  /// Initialize as many devices as possible for this plugin. Devices that fail
-  /// to initialize are ignored.
-  void initDevices(GenericPluginTy &RTL);
-
   /// Return the device presented to the user as device \p DeviceNo if it is
   /// initialized and ready. Otherwise return an error explaining the problem.
   llvm::Expected<DeviceTy &> getDevice(uint32_t DeviceNo);
@@ -117,20 +113,31 @@ struct PluginManager {
     return Devices.getExclusiveAccessor();
   }
 
-  int getNumUsedPlugins() const { return DeviceOffsets.size(); }
-
   // Initialize all plugins.
   void initAllPlugins();
 
   /// Iterator range for all plugins (in use or not, but always valid).
   auto plugins() { return llvm::make_pointee_range(Plugins); }
 
+  /// Iterator range for all plugins (in use or not, but always valid).
+  auto plugins() const { return llvm::make_pointee_range(Plugins); }
+
   /// Return the user provided requirements.
   int64_t getRequirements() const { return Requirements.getRequirements(); }
 
   /// Add \p Flags to the user provided requirements.
   void addRequirements(int64_t Flags) { Requirements.addRequirements(Flags); }
 
+  /// Returns the number of plugins that are active.
+  int getNumActivePlugins() const {
+    int count = 0;
+    for (auto &R : plugins())
+      if (R.is_initialized())
+        ++count;
+
+    return count;
+  }
+
 private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
@@ -138,11 +145,9 @@ private:
   // List of all plugins, in use or not.
   llvm::SmallVector<std::unique_ptr<GenericPluginTy>> Plugins;
 
-  // Mapping of plugins to offsets in the device table.
-  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceOffsets;
-
-  // Mapping of plugins to the number of used devices.
-  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceUsed;
+  // Mapping of plugins to the OpenMP device identifier.
+  llvm::DenseMap<std::pair<const GenericPluginTy *, int32_t>, int32_t>
+      DeviceIds;
 
   // Set of all device images currently in use.
   llvm::DenseSet<const __tgt_device_image *> UsedImages;
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index c6dd954746e4ab1130c675d17a1ff874ee3bf264..663cfdc5fdf0169fca72b98aef00f00c7f6ffa20 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3163,25 +3163,24 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
   uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; }
 
   /// Check whether the image is compatible with an AMDGPU device.
-  Expected<bool> isELFCompatible(StringRef Image) const override {
+  Expected<bool> isELFCompatible(uint32_t DeviceId,
+                                 StringRef Image) const override {
     // Get the associated architecture and flags from the ELF.
     auto ElfOrErr = ELF64LEObjectFile::create(
         MemoryBufferRef(Image, /*Identifier=*/""), /*InitContent=*/false);
     if (!ElfOrErr)
       return ElfOrErr.takeError();
     std::optional<StringRef> Processor = ElfOrErr->tryGetCPUName();
+    if (!Processor)
+      return false;
 
-    for (hsa_agent_t Agent : KernelAgents) {
-      auto TargeTripleAndFeaturesOrError =
-          utils::getTargetTripleAndFeatures(Agent);
-      if (!TargeTripleAndFeaturesOrError)
-        return TargeTripleAndFeaturesOrError.takeError();
-      if (!utils::isImageCompatibleWithEnv(Processor ? *Processor : "",
+    auto TargeTripleAndFeaturesOrError =
+        utils::getTargetTripleAndFeatures(getKernelAgent(DeviceId));
+    if (!TargeTripleAndFeaturesOrError)
+      return TargeTripleAndFeaturesOrError.takeError();
+    return utils::isImageCompatibleWithEnv(Processor ? *Processor : "",
                                            ElfOrErr->getPlatformFlags(),
-                                           *TargeTripleAndFeaturesOrError))
-        return false;
-    }
-    return true;
+                                           *TargeTripleAndFeaturesOrError);
   }
 
   bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override {
@@ -3273,19 +3272,13 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   if (ArgsSize < KernelArgsSize)
     return Plugin::error("Mismatch of kernel arguments size");
 
-  // The args size reported by HSA may or may not contain the implicit args.
-  // For now, assume that HSA does not consider the implicit arguments when
-  // reporting the arguments of a kernel. In the worst case, we can waste
-  // 56 bytes per allocation.
-  uint32_t AllArgsSize = KernelArgsSize + ImplicitArgsSize;
-
   AMDGPUPluginTy &AMDGPUPlugin =
       static_cast<AMDGPUPluginTy &>(GenericDevice.Plugin);
   AMDHostDeviceTy &HostDevice = AMDGPUPlugin.getHostDevice();
   AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager();
 
   void *AllArgs = nullptr;
-  if (auto Err = ArgsMemoryManager.allocate(AllArgsSize, &AllArgs))
+  if (auto Err = ArgsMemoryManager.allocate(ArgsSize, &AllArgs))
     return Err;
 
   // Account for user requested dynamic shared memory.
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index eda6a4fd541e963ae627d0774f29ed8e45dea2fb..88423be039af7479eb653142b38f666769e9ca0d 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -993,11 +993,11 @@ struct GenericPluginTy {
   /// Get the number of active devices.
   int32_t getNumDevices() const { return NumDevices; }
 
-  /// Get the plugin-specific device identifier offset.
-  int32_t getDeviceIdStartIndex() const { return DeviceIdStartIndex; }
-
-  /// Set the plugin-specific device identifier offset.
-  void setDeviceIdStartIndex(int32_t Offset) { DeviceIdStartIndex = Offset; }
+  /// Get the plugin-specific device identifier.
+  int32_t getUserId(int32_t DeviceId) const {
+    assert(UserDeviceIds.contains(DeviceId) && "No user-id registered");
+    return UserDeviceIds.at(DeviceId);
+  }
 
   /// Get the ELF code to recognize the binary image of this plugin.
   virtual uint16_t getMagicElfBits() const = 0;
@@ -1059,7 +1059,8 @@ struct GenericPluginTy {
   /// Indicate if an image is compatible with the plugin devices. Notice that
   /// this function may be called before actually initializing the devices. So
   /// we could not move this function into GenericDeviceTy.
-  virtual Expected<bool> isELFCompatible(StringRef Image) const = 0;
+  virtual Expected<bool> isELFCompatible(uint32_t DeviceID,
+                                         StringRef Image) const = 0;
 
 protected:
   /// Indicate whether a device id is valid.
@@ -1070,11 +1071,18 @@ protected:
 public:
   // TODO: This plugin interface needs to be cleaned up.
 
-  /// Returns true if the plugin has been initialized.
+  /// Returns non-zero if the plugin runtime has been initialized.
   int32_t is_initialized() const;
 
-  /// Returns non-zero if the provided \p Image can be executed by the runtime.
-  int32_t is_valid_binary(__tgt_device_image *Image, bool Initialized = true);
+  /// Returns non-zero if the \p Image is compatible with the plugin. This
+  /// function does not require the plugin to be initialized before use.
+  int32_t is_plugin_compatible(__tgt_device_image *Image);
+
+  /// Returns non-zero if the \p Image is compatible with the device.
+  int32_t is_device_compatible(int32_t DeviceId, __tgt_device_image *Image);
+
+  /// Returns non-zero if the plugin device has been initialized.
+  int32_t is_device_initialized(int32_t DeviceId) const;
 
   /// Initialize the device inside of the plugin.
   int32_t init_device(int32_t DeviceId);
@@ -1180,7 +1188,7 @@ public:
                            const char **ErrStr);
 
   /// Sets the offset into the devices for use by OMPT.
-  int32_t set_device_offset(int32_t DeviceIdOffset);
+  int32_t set_device_identifier(int32_t UserId, int32_t DeviceId);
 
   /// Returns if the plugin can support auotmatic copy.
   int32_t use_auto_zero_copy(int32_t DeviceId);
@@ -1200,10 +1208,8 @@ private:
   /// Number of devices available for the plugin.
   int32_t NumDevices = 0;
 
-  /// Index offset, which when added to a DeviceId, will yield a unique
-  /// user-observable device identifier. This is especially important when
-  /// DeviceIds of multiple plugins / RTLs need to be distinguishable.
-  int32_t DeviceIdStartIndex = 0;
+  /// Map of plugin device identifiers to the user device identifier.
+  llvm::DenseMap<int32_t, int32_t> UserDeviceIds;
 
   /// Array of pointers to the devices. Initially, they are all set to nullptr.
   /// Once a device is initialized, the pointer is stored in the position given
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 913721a15d7131277d767dd37fc4b6788a43593a..5a53c479e33d0da8ff1dcd0a07ed604cac7480b0 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -748,8 +748,7 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (ompt::Initialized) {
     bool ExpectedStatus = false;
     if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
-      performOmptCallback(device_initialize, /*device_num=*/DeviceId +
-                                                 Plugin.getDeviceIdStartIndex(),
+      performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
                           /*type=*/getComputeUnitKind().c_str(),
                           /*device=*/reinterpret_cast<ompt_device_t *>(this),
                           /*lookup=*/ompt::lookupCallbackByName,
@@ -847,9 +846,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
   if (ompt::Initialized) {
     bool ExpectedStatus = true;
     if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
-      performOmptCallback(device_finalize,
-                          /*device_num=*/DeviceId +
-                              Plugin.getDeviceIdStartIndex());
+      performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
   }
 #endif
 
@@ -908,7 +905,7 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
     size_t Bytes =
         getPtrDiff(InputTgtImage->ImageEnd, InputTgtImage->ImageStart);
     performOmptCallback(
-        device_load, /*device_num=*/DeviceId + Plugin.getDeviceIdStartIndex(),
+        device_load, Plugin.getUserId(DeviceId),
         /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
         /*ImgSize=*/Bytes, /*HostAddr=*/InputTgtImage->ImageStart,
         /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
@@ -1492,11 +1489,14 @@ Error GenericDeviceTy::syncEvent(void *EventPtr) {
 bool GenericDeviceTy::useAutoZeroCopy() { return useAutoZeroCopyImpl(); }
 
 Error GenericPluginTy::init() {
+  if (Initialized)
+    return Plugin::success();
+
   auto NumDevicesOrErr = initImpl();
   if (!NumDevicesOrErr)
     return NumDevicesOrErr.takeError();
-
   Initialized = true;
+
   NumDevices = *NumDevicesOrErr;
   if (NumDevices == 0)
     return Plugin::success();
@@ -1517,6 +1517,8 @@ Error GenericPluginTy::init() {
 }
 
 Error GenericPluginTy::deinit() {
+  assert(Initialized && "Plugin was not initialized!");
+
   // Deinitialize all active devices.
   for (int32_t DeviceId = 0; DeviceId < NumDevices; ++DeviceId) {
     if (Devices[DeviceId]) {
@@ -1537,7 +1539,11 @@ Error GenericPluginTy::deinit() {
     delete RecordReplay;
 
   // Perform last deinitializations on the plugin.
-  return deinitImpl();
+  if (Error Err = deinitImpl())
+    return Err;
+  Initialized = false;
+
+  return Plugin::success();
 }
 
 Error GenericPluginTy::initDevice(int32_t DeviceId) {
@@ -1599,8 +1605,7 @@ Expected<bool> GenericPluginTy::checkBitcodeImage(StringRef Image) const {
 
 int32_t GenericPluginTy::is_initialized() const { return Initialized; }
 
-int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image,
-                                         bool Initialized) {
+int32_t GenericPluginTy::is_plugin_compatible(__tgt_device_image *Image) {
   StringRef Buffer(reinterpret_cast<const char *>(Image->ImageStart),
                    target::getPtrDiff(Image->ImageEnd, Image->ImageStart));
 
@@ -1618,11 +1623,43 @@ int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image,
     auto MatchOrErr = checkELFImage(Buffer);
     if (Error Err = MatchOrErr.takeError())
       return HandleError(std::move(Err));
-    if (!Initialized || !*MatchOrErr)
-      return *MatchOrErr;
+    return *MatchOrErr;
+  }
+  case file_magic::bitcode: {
+    auto MatchOrErr = checkBitcodeImage(Buffer);
+    if (Error Err = MatchOrErr.takeError())
+      return HandleError(std::move(Err));
+    return *MatchOrErr;
+  }
+  default:
+    return false;
+  }
+}
+
+int32_t GenericPluginTy::is_device_compatible(int32_t DeviceId,
+                                              __tgt_device_image *Image) {
+  StringRef Buffer(reinterpret_cast<const char *>(Image->ImageStart),
+                   target::getPtrDiff(Image->ImageEnd, Image->ImageStart));
+
+  auto HandleError = [&](Error Err) -> bool {
+    [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
+    DP("Failure to check validity of image %p: %s", Image, ErrStr.c_str());
+    return false;
+  };
+  switch (identify_magic(Buffer)) {
+  case file_magic::elf:
+  case file_magic::elf_relocatable:
+  case file_magic::elf_executable:
+  case file_magic::elf_shared_object:
+  case file_magic::elf_core: {
+    auto MatchOrErr = checkELFImage(Buffer);
+    if (Error Err = MatchOrErr.takeError())
+      return HandleError(std::move(Err));
+    if (!*MatchOrErr)
+      return false;
 
     // Perform plugin-dependent checks for the specific architecture if needed.
-    auto CompatibleOrErr = isELFCompatible(Buffer);
+    auto CompatibleOrErr = isELFCompatible(DeviceId, Buffer);
     if (Error Err = CompatibleOrErr.takeError())
       return HandleError(std::move(Err));
     return *CompatibleOrErr;
@@ -1638,6 +1675,10 @@ int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image,
   }
 }
 
+int32_t GenericPluginTy::is_device_initialized(int32_t DeviceId) const {
+  return isValidDeviceId(DeviceId) && Devices[DeviceId] != nullptr;
+}
+
 int32_t GenericPluginTy::init_device(int32_t DeviceId) {
   auto Err = initDevice(DeviceId);
   if (Err) {
@@ -1985,8 +2026,9 @@ int32_t GenericPluginTy::init_device_info(int32_t DeviceId,
   return OFFLOAD_SUCCESS;
 }
 
-int32_t GenericPluginTy::set_device_offset(int32_t DeviceIdOffset) {
-  setDeviceIdStartIndex(DeviceIdOffset);
+int32_t GenericPluginTy::set_device_identifier(int32_t UserId,
+                                               int32_t DeviceId) {
+  UserDeviceIds[DeviceId] = UserId;
 
   return OFFLOAD_SUCCESS;
 }
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b260334baa18b0015fd574e7b383c412ed3f957a..62460c07415be4d25f542fbc4ff8775e6a7eefc7 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1388,8 +1388,9 @@ struct CUDAPluginTy final : public GenericPluginTy {
 
   const char *getName() const override { return GETNAME(TARGET_NAME); }
 
-  /// Check whether the image is compatible with the available CUDA devices.
-  Expected<bool> isELFCompatible(StringRef Image) const override {
+  /// Check whether the image is compatible with a CUDA device.
+  Expected<bool> isELFCompatible(uint32_t DeviceId,
+                                 StringRef Image) const override {
     auto ElfOrErr =
         ELF64LEObjectFile::create(MemoryBufferRef(Image, /*Identifier=*/""),
                                   /*InitContent=*/false);
@@ -1399,33 +1400,29 @@ struct CUDAPluginTy final : public GenericPluginTy {
     // Get the numeric value for the image's `sm_` value.
     auto SM = ElfOrErr->getPlatformFlags() & ELF::EF_CUDA_SM;
 
-    for (int32_t DevId = 0; DevId < getNumDevices(); ++DevId) {
-      CUdevice Device;
-      CUresult Res = cuDeviceGet(&Device, DevId);
-      if (auto Err = Plugin::check(Res, "Error in cuDeviceGet: %s"))
-        return std::move(Err);
-
-      int32_t Major, Minor;
-      Res = cuDeviceGetAttribute(
-          &Major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, Device);
-      if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s"))
-        return std::move(Err);
-
-      Res = cuDeviceGetAttribute(
-          &Minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, Device);
-      if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s"))
-        return std::move(Err);
-
-      int32_t ImageMajor = SM / 10;
-      int32_t ImageMinor = SM % 10;
-
-      // A cubin generated for a certain compute capability is supported to
-      // run on any GPU with the same major revision and same or higher minor
-      // revision.
-      if (Major != ImageMajor || Minor < ImageMinor)
-        return false;
-    }
-    return true;
+    CUdevice Device;
+    CUresult Res = cuDeviceGet(&Device, DeviceId);
+    if (auto Err = Plugin::check(Res, "Error in cuDeviceGet: %s"))
+      return std::move(Err);
+
+    int32_t Major, Minor;
+    Res = cuDeviceGetAttribute(
+        &Major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, Device);
+    if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s"))
+      return std::move(Err);
+
+    Res = cuDeviceGetAttribute(
+        &Minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, Device);
+    if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s"))
+      return std::move(Err);
+
+    int32_t ImageMajor = SM / 10;
+    int32_t ImageMinor = SM % 10;
+
+    // A cubin generated for a certain compute capability is supported to
+    // run on any GPU with the same major revision and same or higher minor
+    // revision.
+    return Major == ImageMajor && Minor >= ImageMinor;
   }
 };
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index ef84cbaf54588d53046c3d8bfaf1f65ed4d996da..aa59ea618e399ec8728ab990b837e6f6941c8183 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -418,7 +418,9 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   }
 
   /// All images (ELF-compatible) should be compatible with this plugin.
-  Expected<bool> isELFCompatible(StringRef) const override { return true; }
+  Expected<bool> isELFCompatible(uint32_t, StringRef) const override {
+    return true;
+  }
 
   Triple::ArchType getTripleArch() const override {
 #if defined(__x86_64__)
diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp
index 13f08b142b8769890a4f3534800c65f1fa76a415..5e8f91792a551aadbc4c1a0f7f59900ee7c00434 100644
--- a/offload/src/PluginManager.cpp
+++ b/offload/src/PluginManager.cpp
@@ -47,6 +47,9 @@ void PluginManager::deinit() {
   DP("Unloading RTLs...\n");
 
   for (auto &Plugin : Plugins) {
+    if (!Plugin->is_initialized())
+      continue;
+
     if (auto Err = Plugin->deinit()) {
       [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
       DP("Failed to deinit plugin: %s\n", InfoMsg.c_str());
@@ -57,90 +60,15 @@ void PluginManager::deinit() {
   DP("RTLs unloaded!\n");
 }
 
-void PluginManager::initDevices(GenericPluginTy &RTL) {
-  // If this RTL has already been initialized.
-  if (PM->DeviceOffsets.contains(&RTL))
-    return;
-  TIMESCOPE();
-
-  // If this RTL is not already in use, initialize it.
-  assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!");
-
-  // Initialize the device information for the RTL we are about to use.
-  auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
-
-  // Initialize the index of this RTL and save it in the used RTLs.
-  int32_t DeviceOffset = ExclusiveDevicesAccessor->size();
-
-  // Set the device identifier offset in the plugin.
-  RTL.set_device_offset(DeviceOffset);
-
-  int32_t NumberOfUserDevices = 0;
-  int32_t NumPD = RTL.number_of_devices();
-  ExclusiveDevicesAccessor->reserve(DeviceOffset + NumPD);
-  // Auto zero-copy is a per-device property. We need to ensure
-  // that all devices are suggesting to use it.
-  bool UseAutoZeroCopy = !(NumPD == 0);
-  for (int32_t PDevI = 0, UserDevId = DeviceOffset; PDevI < NumPD; PDevI++) {
-    auto Device = std::make_unique<DeviceTy>(&RTL, UserDevId, PDevI);
-    if (auto Err = Device->init()) {
-      DP("Skip plugin known device %d: %s\n", PDevI,
-         toString(std::move(Err)).c_str());
-      continue;
-    }
-    UseAutoZeroCopy = UseAutoZeroCopy && Device->useAutoZeroCopy();
-
-    ExclusiveDevicesAccessor->push_back(std::move(Device));
-    ++NumberOfUserDevices;
-    ++UserDevId;
-  }
-
-  // Auto Zero-Copy can only be currently triggered when the system is an
-  // homogeneous APU architecture without attached discrete GPUs.
-  // If all devices suggest to use it, change requirment flags to trigger
-  // zero-copy behavior when mapping memory.
-  if (UseAutoZeroCopy)
-    addRequirements(OMPX_REQ_AUTO_ZERO_COPY);
-
-  DeviceOffsets[&RTL] = DeviceOffset;
-  DeviceUsed[&RTL] = NumberOfUserDevices;
-  DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset,
-     NumberOfUserDevices, RTL.number_of_devices());
-}
-
 void PluginManager::initAllPlugins() {
-  for (auto &R : Plugins)
-    initDevices(*R);
-}
-
-static void registerImageIntoTranslationTable(TranslationTable &TT,
-                                              int32_t DeviceOffset,
-                                              int32_t NumberOfUserDevices,
-                                              __tgt_device_image *Image) {
-
-  // same size, as when we increase one, we also increase the other.
-  assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
-         "We should have as many images as we have tables!");
-
-  // Resize the Targets Table and Images to accommodate the new targets if
-  // required
-  unsigned TargetsTableMinimumSize = DeviceOffset + NumberOfUserDevices;
-
-  if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
-    TT.DeviceTables.resize(TargetsTableMinimumSize, {});
-    TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
-    TT.TargetsEntries.resize(TargetsTableMinimumSize, {});
-    TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
-  }
-
-  // Register the image in all devices for this target type.
-  for (int32_t I = 0; I < NumberOfUserDevices; ++I) {
-    // If we are changing the image we are also invalidating the target table.
-    if (TT.TargetsImages[DeviceOffset + I] != Image) {
-      TT.TargetsImages[DeviceOffset + I] = Image;
-      TT.TargetsTable[DeviceOffset + I] =
-          0; // lazy initialization of target table.
+  for (auto &R : plugins()) {
+    if (auto Err = R.init()) {
+      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+      DP("Failed to init plugin: %s\n", InfoMsg.c_str());
+      continue;
     }
+    DP("Registered plugin %s with %d visible device(s)\n", R.getName(),
+       R.number_of_devices());
   }
 }
 
@@ -153,27 +81,6 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
     if (Entry.flags == OMP_REGISTER_REQUIRES)
       PM->addRequirements(Entry.data);
 
-  // Initialize all the plugins that have associated images.
-  for (auto &Plugin : Plugins) {
-    // Extract the exectuable image and extra information if availible.
-    for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) {
-      if (Plugin->is_initialized())
-        continue;
-
-      if (!Plugin->is_valid_binary(&Desc->DeviceImages[i],
-                                   /*Initialized=*/false))
-        continue;
-
-      if (auto Err = Plugin->init()) {
-        [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
-        DP("Failed to init plugin: %s\n", InfoMsg.c_str());
-      } else {
-        DP("Registered plugin %s with %d visible device(s)\n",
-           Plugin->getName(), Plugin->number_of_devices());
-      }
-    }
-  }
-
   // Extract the exectuable image and extra information if availible.
   for (int32_t i = 0; i < Desc->NumDeviceImages; ++i)
     PM->addDeviceImage(*Desc, Desc->DeviceImages[i]);
@@ -188,54 +95,110 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
     // Scan the RTLs that have associated images until we find one that supports
     // the current image.
     for (auto &R : PM->plugins()) {
-      if (!R.number_of_devices())
+      if (!R.is_plugin_compatible(Img))
         continue;
 
-      if (!R.is_valid_binary(Img, /*Initialized=*/true)) {
-        DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
-           DPxPTR(Img->ImageStart), R.getName());
-        continue;
+      if (!R.is_initialized()) {
+        if (auto Err = R.init()) {
+          [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+          DP("Failed to init plugin: %s\n", InfoMsg.c_str());
+          continue;
+        }
+        DP("Registered plugin %s with %d visible device(s)\n", R.getName(),
+           R.number_of_devices());
       }
 
-      DP("Image " DPxMOD " is compatible with RTL %s!\n",
-         DPxPTR(Img->ImageStart), R.getName());
-
-      PM->initDevices(R);
+      if (!R.number_of_devices()) {
+        DP("Skipping plugin %s with no visible devices\n", R.getName());
+        continue;
+      }
 
-      // Initialize (if necessary) translation table for this library.
-      PM->TrlTblMtx.lock();
-      if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) {
-        PM->HostEntriesBeginRegistrationOrder.push_back(Desc->HostEntriesBegin);
-        TranslationTable &TransTable =
+      for (int32_t DeviceId = 0; DeviceId < R.number_of_devices(); ++DeviceId) {
+        if (!R.is_device_compatible(DeviceId, Img))
+          continue;
+
+        DP("Image " DPxMOD " is compatible with RTL %s device %d!\n",
+           DPxPTR(Img->ImageStart), R.getName(), DeviceId);
+
+        if (!R.is_device_initialized(DeviceId)) {
+          // Initialize the device information for the RTL we are about to use.
+          auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
+
+          int32_t UserId = ExclusiveDevicesAccessor->size();
+
+          // Set the device identifier offset in the plugin.
+#ifdef OMPT_SUPPORT
+          R.set_device_identifier(UserId, DeviceId);
+#endif
+
+          auto Device = std::make_unique<DeviceTy>(&R, UserId, DeviceId);
+          if (auto Err = Device->init()) {
+            [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+            DP("Failed to init device %d: %s\n", DeviceId, InfoMsg.c_str());
+            continue;
+          }
+
+          ExclusiveDevicesAccessor->push_back(std::move(Device));
+
+          // We need to map between the plugin's device identifier and the one
+          // that OpenMP will use.
+          PM->DeviceIds[std::make_pair(&R, DeviceId)] = UserId;
+        }
+
+        // Initialize (if necessary) translation table for this library.
+        PM->TrlTblMtx.lock();
+        if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) {
+          PM->HostEntriesBeginRegistrationOrder.push_back(
+              Desc->HostEntriesBegin);
+          TranslationTable &TT =
+              (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
+          TT.HostTable.EntriesBegin = Desc->HostEntriesBegin;
+          TT.HostTable.EntriesEnd = Desc->HostEntriesEnd;
+        }
+
+        // Retrieve translation table for this library.
+        TranslationTable &TT =
             (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
-        TransTable.HostTable.EntriesBegin = Desc->HostEntriesBegin;
-        TransTable.HostTable.EntriesEnd = Desc->HostEntriesEnd;
-      }
 
-      // Retrieve translation table for this library.
-      TranslationTable &TransTable =
-          (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
+        DP("Registering image " DPxMOD " with RTL %s!\n",
+           DPxPTR(Img->ImageStart), R.getName());
 
-      DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart),
-         R.getName());
+        auto UserId = PM->DeviceIds[std::make_pair(&R, DeviceId)];
+        if (TT.TargetsTable.size() < static_cast<size_t>(UserId + 1)) {
+          TT.DeviceTables.resize(UserId + 1, {});
+          TT.TargetsImages.resize(UserId + 1, nullptr);
+          TT.TargetsEntries.resize(UserId + 1, {});
+          TT.TargetsTable.resize(UserId + 1, nullptr);
+        }
 
-      registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R],
-                                        PM->DeviceUsed[&R], Img);
-      PM->UsedImages.insert(Img);
+        // Register the image for this target type and invalidate the table.
+        TT.TargetsImages[UserId] = Img;
+        TT.TargetsTable[UserId] = nullptr;
 
-      PM->TrlTblMtx.unlock();
-      FoundRTL = &R;
+        PM->UsedImages.insert(Img);
+        FoundRTL = &R;
 
-      // if an RTL was found we are done - proceed to register the next image
-      break;
+        PM->TrlTblMtx.unlock();
+      }
     }
-
-    if (!FoundRTL) {
+    if (!FoundRTL)
       DP("No RTL found for image " DPxMOD "!\n", DPxPTR(Img->ImageStart));
-    }
   }
   PM->RTLsMtx.unlock();
 
+  bool UseAutoZeroCopy = Plugins.size() > 0;
+
+  auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
+  for (const auto &Device : *ExclusiveDevicesAccessor)
+    UseAutoZeroCopy &= Device->useAutoZeroCopy();
+
+  // Auto Zero-Copy can only be currently triggered when the system is an
+  // homogeneous APU architecture without attached discrete GPUs.
+  // If all devices suggest to use it, change requirment flags to trigger
+  // zero-copy behavior when mapping memory.
+  if (UseAutoZeroCopy)
+    addRequirements(OMPX_REQ_AUTO_ZERO_COPY);
+
   DP("Done registering entries!\n");
 }
 
@@ -257,7 +220,7 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
     // Scan the RTLs that have associated images until we find one that supports
     // the current image. We only need to scan RTLs that are already being used.
     for (auto &R : PM->plugins()) {
-      if (!DeviceOffsets.contains(&R))
+      if (R.is_initialized())
         continue;
 
       // Ensure that we do not use any unused images associated with this RTL.
diff --git a/offload/src/omptarget.cpp b/offload/src/omptarget.cpp
index 91e1213f175ee9747b86d9ba21ff6b6331024d81..9bca8529c5ee37b7a254c81deaa5a38b8951b965 100644
--- a/offload/src/omptarget.cpp
+++ b/offload/src/omptarget.cpp
@@ -315,7 +315,7 @@ void handleTargetOutcome(bool Success, ident_t *Loc) {
         FAILURE_MESSAGE("Consult https://openmp.llvm.org/design/Runtimes.html "
                         "for debugging options.\n");
 
-      if (!PM->getNumUsedPlugins()) {
+      if (!PM->getNumActivePlugins()) {
         FAILURE_MESSAGE(
             "No images found compatible with the installed hardware. ");
 
diff --git a/offload/test/offloading/ompx_bare_shfl_down_sync.cpp b/offload/test/offloading/ompx_bare_shfl_down_sync.cpp
index d2569a5b02665e4def823886574beed0fcd78f23..c9246894b0894e44a0b06ebec8de9079e65ac283 100644
--- a/offload/test/offloading/ompx_bare_shfl_down_sync.cpp
+++ b/offload/test/offloading/ompx_bare_shfl_down_sync.cpp
@@ -23,7 +23,7 @@ bool equal(T LHS, T RHS) {
 template <typename T,
           std::enable_if_t<std::is_floating_point<T>::value, bool> = true>
 bool equal(T LHS, T RHS) {
-  return std::abs(LHS - RHS) < std::numeric_limits<T>::epsilon();
+  return __builtin_fabs(LHS - RHS) < std::numeric_limits<T>::epsilon();
 }
 
 template <typename T> void test() {
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index c228a392e4c7aeb250fbb9c3dc4b25e9e3706bdd..3b4259dfa380e892a362674c77185f383860b494 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -13,6 +13,8 @@ list(INSERT CMAKE_MODULE_PATH 0
 if (OPENMP_STANDALONE_BUILD OR "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   set(OPENMP_STANDALONE_BUILD TRUE)
   project(openmp C CXX ASM)
+else()
+  set(OPENMP_STANDALONE_BUILD FALSE)
 endif()
 
 # Must go below project(..)
@@ -137,8 +139,10 @@ if (OPENMP_ENABLE_OMPT_TOOLS)
 endif()
 
 # Propagate OMPT support to offload
-set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE)
-set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE)
+if(NOT ${OPENMP_STANDALONE_BUILD})
+  set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE)
+  set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE)
+endif()
 
 option(OPENMP_MSVC_NAME_SCHEME "Build dll with MSVC naming scheme." OFF)
 
diff --git a/third-party/unittest/googletest/include/gtest/internal/gtest-port.h b/third-party/unittest/googletest/include/gtest/internal/gtest-port.h
index a17349e401503da4de426ed04433c929909e36ec..02e1eb0a914f2167cb0c8094b0aa392e62ca7a4e 100644
--- a/third-party/unittest/googletest/include/gtest/internal/gtest-port.h
+++ b/third-party/unittest/googletest/include/gtest/internal/gtest-port.h
@@ -652,7 +652,7 @@ typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION;
 // Determines whether to support death tests.
 // pops up a dialog window that cannot be suppressed programmatically.
 #if (defined(GTEST_OS_LINUX) || defined(GTEST_OS_CYGWIN) ||           \
-     defined(GTEST_OS_SOLARIS) ||                                     \
+     defined(GTEST_OS_SOLARIS) || defined(GTEST_OS_ZOS) ||            \
      (defined(GTEST_OS_MAC) && !defined(GTEST_OS_IOS)) ||             \
      (defined(GTEST_OS_WINDOWS_DESKTOP) && _MSC_VER) ||               \
      defined(GTEST_OS_WINDOWS_MINGW) || defined(GTEST_OS_AIX) ||      \
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index f3809bd74814072084d73b8f6741fa6f2994ecc2..ea3d5c2f5816e6a6c2265fa694bddc356d43e4f1 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1111,7 +1111,7 @@ libc_support_library(
     ],
     defines = [
         "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY",
-        "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT"
+        "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT",
     ],
     target_compatible_with = select({
         "@platforms//os:linux": [],
@@ -1119,9 +1119,9 @@ libc_support_library(
     }),
     deps = [
         ":__support_cpp_optional",
-        ":__support_time_linux",
         ":__support_threads_linux_futex_utils",
         ":__support_threads_sleep",
+        ":__support_time_linux",
         ":types_pid_t",
     ],
 )
@@ -3399,7 +3399,7 @@ libc_function(
 
 libc_function(
     name = "printf",
-    srcs = ["src/stdio/printf.cpp"],
+    srcs = ["src/stdio/generic/printf.cpp"],
     hdrs = ["src/stdio/printf.h"],
     deps = [
         ":__support_arg_list",
@@ -3448,7 +3448,7 @@ libc_function(
 
 libc_function(
     name = "vprintf",
-    srcs = ["src/stdio/vprintf.cpp"],
+    srcs = ["src/stdio/generic/vprintf.cpp"],
     hdrs = ["src/stdio/vprintf.h"],
     deps = [
         ":__support_arg_list",
@@ -3580,6 +3580,7 @@ libc_function(
     }),
     weak = True,
     deps = [
+        ":__support_macros_sanitizer",
         ":__support_osutil_syscall",
         ":errno",
         ":hdr_signal_macros",
@@ -3599,6 +3600,7 @@ libc_function(
     }),
     weak = True,
     deps = [
+        ":__support_macros_sanitizer",
         ":__support_osutil_syscall",
         ":errno",
         ":hdr_signal_macros",
@@ -3620,6 +3622,7 @@ libc_function(
 #     }),
 #     weak = True,
 #     deps = [
+#         ":__support_macros_sanitizer",
 #         ":__support_osutil_syscall",
 #         ":errno",
 #         ":hdr_signal_macros",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
index 3980ef60c197eda379cf8fd1fa61b76b9a0b5164..c8001fe1e5812481e37675c6fc2498b545ffce96 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
@@ -99,6 +99,7 @@ libc_test(
     name = "fixedvector_test",
     srcs = ["fixedvector_test.cpp"],
     deps = [
+        "//libc:__support_cpp_array",
         "//libc:__support_fixedvector",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
index 4f72a0a8e1863a6a1638136ed4e70cbd71436438..fac692addb9e5a2f061c77783e6b228db764935d 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
@@ -529,7 +529,9 @@ libc_support_library(
         "//libc:__support_cpp_bit",
         "//libc:__support_cpp_type_traits",
         "//libc:__support_fputil_basic_operations",
+        "//libc:__support_fputil_fenv_impl",
         "//libc:__support_fputil_fp_bits",
+        "//libc:hdr_fenv_macros",
         "//libc:hdr_math_macros",
         "//libc/test/UnitTest:LibcUnitTest",
         "//libc/test/UnitTest:fp_test_helpers",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
index 11d2e5aa340f6618824fb3b3c3c62249ba99d85b..9ec3a5e5ad57a7c2a73fbb7ee6fd3b91e3f464a7 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
@@ -25,6 +25,7 @@ def math_test(name, hdrs = [], deps = [], **kwargs):
         srcs = [test_name + ".cpp"] + hdrs,
         libc_function_deps = ["//libc:func_name".replace("func_name", name)],
         deps = [
+            "//libc/test/UnitTest:fp_test_helpers",
             "//libc:__support_cpp_algorithm",
             "//libc:__support_fputil_basic_operations",
             "//libc:__support_fputil_fenv_impl",
@@ -32,9 +33,9 @@ def math_test(name, hdrs = [], deps = [], **kwargs):
             "//libc:__support_fputil_manipulation_functions",
             "//libc:__support_fputil_nearest_integer_operations",
             "//libc:__support_fputil_normal_float",
+            "//libc:__support_macros_properties_architectures",
             "//libc:__support_math_extras",
             "//libc:__support_uint128",
-            "//libc/test/UnitTest:fp_test_helpers",
             "//libc:hdr_math_macros",
         ] + deps,
         **kwargs
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
index 7d4b9978db3f25bcd6150e3c609da9218c5e82d9..2ad2209925cebf5c8de1adf804f4964e374e4167 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
@@ -117,6 +117,7 @@ math_test(
     name = "llrintf128",
     hdrs = ["RoundToIntegerTest.h"],
 )
+
 math_test(
     name = "lroundf128",
     hdrs = ["RoundToIntegerTest.h"],
@@ -135,7 +136,9 @@ libc_support_library(
         "//libc:__support_cpp_bit",
         "//libc:__support_cpp_type_traits",
         "//libc:__support_fputil_basic_operations",
+        "//libc:__support_fputil_fenv_impl",
         "//libc:__support_fputil_fp_bits",
+        "//libc:hdr_fenv_macros",
         "//libc:hdr_math_macros",
         "//libc/test/UnitTest:LibcUnitTest",
         "//libc/test/UnitTest:fp_test_helpers",
diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
index 53a8c9b9476f3eb3e0711aa5eb7ade01ce623168..6dd1fc4679d43465f49531528d53ca0b9f6fcba6 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
@@ -46,6 +46,7 @@ libc_support_library(
         "//libc:__support_cpp_type_traits",
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_fputil_fpbits_str",
+        "//libc:__support_macros_properties_types",
         "//libc:hdr_math_macros",
         "//libc/test/UnitTest:LibcUnitTest",
         "//libc/test/UnitTest:fp_test_helpers",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 2ffbc783ec972f350d8aac2212511c7efa6b95a0..cf7ee18f0d0688f808533dc4cf706010df3c17af 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -164,6 +164,11 @@ td_library(
     includes = ["include"],
 )
 
+llvm_config_target_defines = [
+    "LLVM_HAS_{}_TARGET=1".format(t)
+    for t in llvm_targets
+]
+
 cc_library(
     name = "config",
     hdrs = [
@@ -171,7 +176,7 @@ cc_library(
         "include/llvm/Config/llvm-config.h",
     ],
     copts = llvm_copts,
-    defines = llvm_config_defines,
+    defines = llvm_config_defines + llvm_config_target_defines,
     includes = ["include"],
     textual_hdrs = [
         "include/llvm/Config/AsmParsers.def",
@@ -1475,6 +1480,7 @@ cc_library(
     deps = [
         ":Analysis",
         ":Core",
+        ":Scalar",
         ":Support",
         ":Target",
         ":TransformUtils",
@@ -1755,6 +1761,7 @@ cc_library(
         ":TransformUtils",
         ":Vectorize",
         ":config",
+        ":ir_headers",
     ],
 )